python三高问题分析

1.高内存

模拟高内存场景:利用两张表df1和df2以第一列做为连接键进行外关联,每张表中作为连接键的列相同的数据越是多,

在进行外关联时形成新表数据就会成指数积增长

import pandas as pd

data1 = [
    ["a", 0],
    ["a", 1],
    ["a", 2],
    ["b", 3],
    ["b", 4],
    ["b", 5],

]
data2 = [
    ["a", 6],
    ["a", 7],
    ["a", 8],
    ["b", 9],
    ["b", 10],
    ["b", 11],
]

df1 = pd.DataFrame(data1)
print("----------------df1----------------")
print(df1)

df2 = pd.DataFrame(data2)
print("----------------df2----------------")
print(df2)

df3 = pd.merge(df1, df2, on=[0], how="outer")
print("----------------d3----------------")
print(df3)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

结果

----------------df1----------------
   0  1
0  a  0
1  a  1
2  a  2
3  b  3
4  b  4
5  b  5
----------------df2----------------
   0   1
0  a   6
1  a   7
2  a   8
3  b   9
4  b  10
5  b  11
----------------d3----------------
    0  1_x  1_y
0   a    0    6
1   a    0    7
2   a    0    8
3   a    1    6
4   a    1    7
5   a    1    8
6   a    2    6
7   a    2    7
8   a    2    8
9   b    3    9
10  b    3   10
11  b    3   11
12  b    4    9
13  b    4   10
14  b    4   11
15  b    5    9
16  b    5   10
17  b    5   11

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

分析

import linecache
import signal
import tracemalloc


def display_top(snapshot, key_type='lineno', limit=10):
    snapshot = snapshot.filter_traces((
        tracemalloc.Filter(False, ""),
        tracemalloc.Filter(False, ""),
    ))
    top_stats = snapshot.statistics(key_type)

    print("Top %s lines" % limit)
    for index, stat in enumerate(top_stats[:limit], 1):
        frame = stat.traceback[0]
        print("#%s: %s:%s: %.1f KiB"
              % (index, frame.filename, frame.lineno, stat.size / 1024))
        line = linecache.getline(frame.filename, frame.lineno).strip()
        if line:
            print('    %s' % line)

    other = top_stats[limit:]
    if other:
        size = sum(stat.size for stat in other)
        print("%s other: %.1f KiB" % (len(other), size / 1024))
    total = sum(stat.size for stat in top_stats)
    print("Total allocated size: %.1f KiB" % (total / 1024))


def handler():
    snapshot = tracemalloc.take_snapshot()
    display_top(snapshot, key_type='traceback')


# 注册信号,因为会在调用过程中升到100%,所以在此之前停止
signal.signal(signal.SIGINT, handler)
tracemalloc.start()
# ... run your application ...

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

2.高网络延时

cprofile:分析哪一个功能模块效率低,延迟高,占用资源多

方式一

import cProfile


def fib():
    ...


cProfile.run('fib')
1
2
3
4
5
6
7
8

方式二

python3 -m cProfile xxx.py
1

说明

traceroute

traceroute --tcp -p 端口 -n 目的ip
1

3.高cpu

1.top命令查看cpu使用情况

2.ps -aux 显示所有包含其他使用者的进程

3.使用abs测试linux的性能

ab -c 10 -n 100 http://192.168.0.10:10000/
1

4.使用pef分析调用关系

#-g 开启调用关系分析 -p 指定php-fpm的进程号2151
perf top -g -p 21515
1
2

5.查询某个目录下的具体文本

grep 具体文本 -r app
1