算法测试流程脚本工具站
🍉一、录入sql和批量上传minIo图片，
录入sql（掠过）...,
一个上传，一个下载，只需上传，找到
def upload():
# NOTE：Step:3 把拉下来的图片传上去给XXX服务器的minio
up_data_minio("test", image_Urls_path='res')修改image_Urls_path，为批量上传图片目录位置。
找到
# 上传图片到minio
def up_data_minio(bucket: str, image_Urls_path='imageUrlFromminIO'):
# TODO：minio_conf唯一要修改的地方！
minio_conf = {
'endpoint': '10.168.1.96:9000',
# 'endpoint': '172.20.0.67:9000',
'access_key': 'minioadmin',
# 'access_key': 'root',
'secret_key': 'minioadmin',
# 'secret_key': 'Lidar4d@#321',
'secure': False
}修改minio_conf为需要上传到指定minIO的绑定信息。
import minio
import pymysql
import openpyxl
import os
def get_data_from_mysql():
# 连接数据库-
conn = pymysql.connect(host="10.168.1.94", user="root", passwd="root", db="RemotePatrolDB", port=3369, charset="utf8")
cur = conn.cursor() # 创建游标对象
# 查询表中数据
cur.execute("SELECT * FROM CorrectPoint_fushun;")
df = cur.fetchall() # 获取所有数据
imageUrls = []
for data in df:
imageUrls.append(data[15])
cur.close()
conn.close()
return imageUrls
def save_for_excel(df):
wb = openpyxl.Workbook()
ws = wb.active
for row in df:
ws.append(row)
wb.save("文件名.xlsx")
# 从minio上面拉取图片
def load_data_minio(bucket: str, imageUrls):
minio_conf = {
'endpoint': '10.168.1.96:9000',
'access_key': 'minioadmin',
'secret_key': 'minioadmin',
'secure': False
}
client = minio.Minio(**minio_conf)
if not client.bucket_exists(bucket):
return None
root_path = os.path.join("imageUrlFromminIO")
for imageUrl in imageUrls:
imageUrl = imageUrl.split('/')[-1]
data = client.get_object(bucket, imageUrl)
save_path = os.path.join(root_path, imageUrl)
with open(save_path, 'wb') as file_data:
for d in data.stream(32 * 1024):
file_data.write(d)
return data.data
# 上传图片到minio
def up_data_minio(bucket: str, image_Urls_path='imageUrlFromminIO'):
# TODO：minio_conf唯一要修改的地方！
minio_conf = {
'endpoint': '10.168.1.96:9000',
# 'endpoint': '172.20.0.67:9000',
'access_key': 'minioadmin',
# 'access_key': 'root',
'secret_key': 'minioadmin',
# 'secret_key': 'Lidar4d@#321',
'secure': False
}
for im_name in os.listdir(image_Urls_path):
client = minio.Minio(**minio_conf)
print(im_name, "load sucessed!")
'''
client.fput_object('mybucket', 'myobject.jpg', '/path/to/myobject.jpg', content_type='image/jpeg')
'''
client.fput_object(bucket_name=bucket, object_name=im_name,
file_path=os.path.join(image_Urls_path, im_name),
content_type='image/jpeg'
)
def download():
# NOTE：Step:1 拉取数据库信息
imageUrls = get_data_from_mysql()
print(imageUrls)
# NOTE：Step:2 把图片从96的minio上面拉下来
print(type(load_data_minio("test", imageUrls)))
def upload():
# NOTE：Step:3 把拉下来的图片传上去给XXX服务器的minio
up_data_minio("test", image_Urls_path='res')
if __name__ == "__main__":
# 子良使用
# download()
# 豪哥使用
upload()
'''
用于批量修改数据库ImagePath字段信息，替换为自己的ip。
---
UPDATE CorrectPoint SET ImagePath=REPLACE(ImagePath, '192.168.120.188', '10.168.1.96');
UPDATE CorrectPoint SET ImagePath=REPLACE(ImagePath, '10.168.1.96', '192.168.120.188');
UPDATE CorrectPoint_lanjiang SET ImagePath=(REPLACE(ImagePath,"/intelligentremotepatrol/obj/","/test/"));
'''
🍉二、自动分区指针、数显、油面、状态数据
刚拿到数据的时候，数据目录是这样的：
可以看到，里面有数显、也有指针类表计，当然还有更多的没有展示出来：油面表和状态类。
这样就很痛苦，因为每个类别对应的算法不同，想要批量测试，必须将他们分区，所以我们可以运行下面的代码：
1. 连接数据库。
2.找到数据库中对应点位的算法类别标签。
3.根据标签将每一个数据分到指定分区。
import minio
import pymysql
import openpyxl
import os
import shutil
def get_data_from_mysql():
# 连接数据库-
conn = pymysql.connect(host="10.168.1.94", user="root", passwd="root", db="RemotePatrolDB", port=3369, charset="utf8")
cur = conn.cursor() # 创建游标对象
# 查询表中数据
cur.execute("SELECT * FROM CorrectPoint_fushun;")
df = cur.fetchall() # 获取所有数据
imageUrls = {}
for data in df:
imageUrls[data[1]] = data[5]
cur.close()
conn.close()
return imageUrls
if __name__ == "__main__":
img_list = os.listdir('./')
id_list = [temp.split('.')[0].split('_')[-1] for temp in img_list]
meter_type = get_data_from_mysql()
is_null_ids = []
is_v5 = []
is_paddleocr = []
is_ywj = []
if not os.path.exists('meter_ywj'):
os.makedirs('meter_ywj')
if not os.path.exists('meter_v5'):
os.makedirs('meter_v5')
if not os.path.exists('paddleocr'):
os.makedirs('paddleocr')
if not os.path.exists('zt_shibie'):
os.makedirs('zt_shibie')
for ind, i in enumerate(id_list):
if i == 'not':
continue
if i not in meter_type:
is_null_ids.append(i)
continue
if meter_type[i] == 'meter_v5':
shutil.move(img_list[ind], 'meter_v5/' + img_list[ind])
elif meter_type[i] == 'paddleocr':
shutil.move(img_list[ind], 'paddleocr/' + img_list[ind])
elif meter_type[i] == 'meter_ywj':
shutil.move(img_list[ind], 'meter_ywj/' + img_list[ind])
print("未标定的点位：", is_null_ids)结果如下：
🍉三、自动筛选出未识别出的点位，并且放到一个单独的分区，可以逐个测试，找到每一个测试失败的原因
import os
import shutil
def find(dir_path):
ori_data = set(os.listdir(dir_path))
res_data = set(os.listdir(dir_path + '/' + 'res'))
results = []
for ori in ori_data:
if ori == 'res':
continue
ori_id = ori.split('_')[0] + '.jpg'
if not ori_id in res_data and not ori in res_data:
results.append(ori)
# shutil.move(os.path.join(dir_path, ori), ori)
# shutil.copyfile(os.path.join(dir_path, ori), ori)
print(results)
print("len(results):", len(results))
if __name__ == "__main__":
find('lanjiang_meter1226')
🍉四、 找到数据库中录入失败的点位
import minio
import pymysql
import openpyxl
import os
import shutil
def get_column_from_mysql():
# 连接到MySQL数据库
connection = pymysql.connect(
host="10.168.1.94",
user="root",
password="root",
database="RemotePatrolDB",
port=3369,
charset="utf8",
cursorclass=pymysql.cursors.DictCursor
)
try:
# 创建游标对象
with connection.cursor() as cursor:
# 执行查询表结构的SQL语句
table_name = 'CorrectPoint_lanjiang'
query = f"DESCRIBE {table_name}"
cursor.execute(query)
# 获取查询结果
result = cursor.fetchall()
# 打印表的所有字段
for i,row in enumerate(result):
print(i, row['Field'])
finally:
# 关闭数据库连接
connection.close()
def get_data_from_mysql():
get_column_from_mysql()
# 连接数据库-
conn = pymysql.connect(
host="10.168.1.94",
user="root",
password="root",
database="RemotePatrolDB",
port=3369,
charset="utf8",
cursorclass=pymysql.cursors.DictCursor
)
cur = conn.cursor() # 创建游标对象
# 查询表中数据
cur.execute("SELECT * FROM CorrectPoint_lanjiang;")
df = cur.fetchall() # 获取所有数据
imageUrls = []
ids = []
for data in df:
ids.append(data['PointPositionId'])
if len(data['ImagePath']) > 0:
continue
imageUrls.append(data['PointPositionId'])
cur.close()
conn.close()
return ids, imageUrls
def find_notLoad_id(ids, image_paths='lanjiang_meter1226'):
img_list = os.listdir(image_paths)
results = []
for im in img_list:
if im == 'res':
continue
id = im.split('_')[0]
if not id in ids:
results.append(im)
return results
if __name__ == "__main__":
ids, imageUrls = get_data_from_mysql()
print('***************************')
print("【警告】minIO为空的图片：")
print('***************************')
print(imageUrls)
print("len(imageUrls):", len(imageUrls))
print('***************************')
print("【警告】未录入点位的图片：")
print('***************************')
ids_error = find_notLoad_id(ids, image_paths='lanjiang_meter1226')
print(ids_error)
print("len(ids_error):", len(ids_error))
0 Id
1 PointPositionId
2 Position
3 MinValue
4 MaxValue
5 AlgorithmType
6 CreatorId
7 CreatorName
8 CreatorTime
9 UpdateId
10 UpdateName
11 UpdateTime
12 IsDelete
13 ImagePath
14 FrameValue
15 MarkType
16 Type
17 RegionCoordinate
18 RowNum
19 ColumnNum
20 Time
21 Tags
***************************
【警告】minIO为空的图片：
***************************
['3a0ce56d-2746-2f58-46f2-aa56f24b2324', '3a0ce56d-2746-4ca1-cb92-9a8446883521', '3a0ce56d-2746-36c7-2b65-24159b4bcab8', '3a0ce56d-2746-c90c-dac5-47fca3e28ee5', '3a0ce56d-2747-9700-e48c-b771397bb369', '3a0d7153-5342-1877-54e7-98a79fe1a23f', '3a0d7153-5343-5750-300f-3d379c106df2', '3a0d7153-5343-6063-9c42-a9ca6af7c961', '3a0d7153-5344-6f60-3753-15e9f6d2f35b', '3a0d7153-5343-f060-b02f-6f15924d27d4', '3a0d7153-5344-babb-fafd-71e4a7148114', '3a0d7153-5344-d702-763e-1800629853f3', '3a0d7153-5344-46fb-bc2f-0ce786508e4b', '3a0d7153-5344-bde3-31c8-f421afb3ad6b', '3a0d7153-5344-1f4c-de63-97fbdf9c33fb', '3a0d7153-5344-c48a-c61a-6d5ed9180d9d', '3a0d7153-5344-d35a-0482-25a5cc66e46f', '3a0d7153-5344-e983-3246-9ca182933208', '3a0d7153-5344-d41a-51ff-1609dbb9af79', '3a0d7153-5344-c173-c0ef-59e0eb3b24aa', '3a0ee406-d07b-9c19-9270-32c2f02befa8', '3a0ee406-d07c-021f-97ad-652b00de6102', '3a0ee406-d07c-0247-7c15-057b9407fb0e', '3a0ee406-d07c-27a4-876f-6dac0d737f68', '3a0ee406-d07d-4b54-6931-072ac4cf7538', '3a0ee406-d07f-6421-43c7-77f6c8d8ff1f', '3a0f0427-d532-0530-134f-a8eb01640ec8', '3a0f0427-d533-254f-6a87-b4eb500c4581', '3a0f0427-d533-8b2e-2279-a6f38f592545', '3a0f0427-d532-a738-89a6-af4af7328446', '3a0f02fd-a129-cde9-6e89-58662a3c6cf4', '3a0f0427-d536-1f1f-0ad7-14fc9a6c0094', '3a0f0427-d536-4b05-6640-791df6aa7c3b', '3a0f64db-4e12-af0a-5fa8-65100a5fddd9', '3a0f06ed-daa4-99e2-e3fa-b925ba024921']
len(imageUrls): 35
***************************
【警告】未录入点位的图片：
***************************
['3a0efe65-3f13-42a3-6098-aebbc07e9a88_3a0fb216-76d7-3764-295b-dc7093a55ef6_20231225590349.jpg', '3a0efe65-3f14-5f10-d5c0-ba72d79b6893_3a0fb216-76d8-7a46-7b6c-58821c0595b2_20231225060447.jpg', '3a0efe65-3f19-678d-edf4-a804c300a51a_3a0fb216-76d7-f405-57fa-fb0999b37238_20231225070450.jpg', '3a0f0211-b2d8-61e5-5533-764e2ead9f2c_3a0fb216-76d5-a2e8-4992-62c198d56478_20231225040411.jpg', '3a0f0211-b2d8-648a-15cb-6465c1d4cb91_3a0fb216-76d5-a2e8-4992-62c198d56478_20231225040446.jpg', '3a0f0211-b2d8-7ff2-afbb-ac10399d0e54_3a0fb216-76d5-a2e8-4992-62c198d56478_20231225040423.jpg', '3a0f0211-b2d8-9184-80fc-a62f23fe5975_3a0fb216-76db-8134-20c1-1f3960819ff3_20231225060411.jpg', '3a0f0211-b2d8-af28-3ef8-4a17141059b4_3a0fb216-76db-8134-20c1-1f3960819ff3_20231225050426.jpg', '3a0f0211-b2d8-bcf1-f581-d7b774248d92_3a0fb216-76d5-a2e8-4992-62c198d56478_20231225040400.jpg', '3a0f0211-b2d8-bf27-af5d-f17e7a1bcdd7_3a0fb216-76db-8134-20c1-1f3960819ff3_20231225060400.jpg', '3a0f0211-b2d8-e2af-32d8-2c049edea555_3a0fb216-76d5-a2e8-4992-62c198d56478_20231225030447.jpg', '3a0f0211-b2d8-f5d4-4959-8bcad63ca3ea_3a0fb216-76d5-a2e8-4992-62c198d56478_20231225040434.jpg', '3a0f0211-b2da-0781-6d31-b4cdb8b9df02_3a0fb216-76d5-40e9-8f4d-ef8d9dedeb45_20231225000449.jpg', '3a0f0211-b2db-1967-de35-079f3765b631_3a0fb216-76db-5277-bfbd-b6583952ce68_20231225060423.jpg', '3a0f0211-b2db-2dcd-585a-2abad0c31b1b_3a0fb216-76db-e24d-d80b-02206c47655c_20231225590326.jpg', '3a0f0211-b2db-3a64-4a09-ad834ff42571_3a0fb216-76db-e24d-d80b-02206c47655c_20231225580352.jpg', '3a0f0211-b2db-4377-69ba-557d58642223_3a0fb216-76db-e24d-d80b-02206c47655c_20231225590338.jpg', '3a0f0211-b2db-60f0-c6ac-c796bbb144fe_3a0fb216-76db-e24d-d80b-02206c47655c_20231225590315.jpg', '3a0f0211-b2db-81a5-446b-4a98683da222_3a0fb216-76db-e24d-d80b-02206c47655c_20231225590303.jpg', '3a0f0427-d52d-a45c-cdb4-1a2010190084_3a0fb216-76d7-e192-15be-5e7718b8eced_20231225070445.jpg', '3a0f06ed-daa4-3209-c85d-438478893957_3a0fb216-76da-c262-5fc3-d091cd39c381_20231225590304.jpg', '3a0f06ed-daa4-47b4-7d04-5afb9db6f351_3a0fb216-76d4-5546-c271-517f7f4bd2b9_20231225090428.jpg', '3a0f06ed-daa4-e789-2cb1-be0df84590af_3a0fb216-76d4-5546-c271-517f7f4bd2b9_20231225090439.jpg', '3a0f0716-eedd-0665-6153-2bfd4752bd5f_3a0fb216-76d5-c63c-f2e5-85d5a4ba55c3_20231225100418.jpg', '3a0f0716-eedd-8084-fb19-05aeb9e9703f_3a0fb216-76d5-c63c-f2e5-85d5a4ba55c3_20231225100406.jpg', '3a0f0716-eedd-9599-b03f-7abc46a0f550_3a0fb216-76d5-c63c-f2e5-85d5a4ba55c3_20231225090442.jpg', '3a0f0716-eedd-ac1a-d551-5c56abdef0a4_3a0fb216-76d5-c63c-f2e5-85d5a4ba55c3_20231225090455.jpg', '3a0f0716-eedd-ade1-670b-64dc55474505_3a0fb216-76d5-c63c-f2e5-85d5a4ba55c3_20231225090418.jpg', '3a0f5a44-a13a-17ee-980f-3915ddf9bbbb_3a0fb216-76d0-a4d5-9563-58bb7e7e5792_20231225080429.jpg', '3a0f5a44-a13b-0a71-398f-f8a8ff4eea9f_3a0fb216-76cd-dfbb-4281-0d3ebc3fc6e8_20231225050450.jpg', '3a0f5a44-a13b-1215-c2ad-955100c6d158_3a0fb216-76d7-e192-15be-5e7718b8eced_20231225090428.jpg', '3a0f5a44-a13b-24c4-f5a3-744a21e7f7b6_3a0fb216-76d9-0ed1-b5ae-0165bd60b3d4_20231225090448.jpg', '3a0f5a44-a13b-34ca-f87d-c7d429e110c9_3a0fb216-76d9-149d-6b9e-9aeca820cc2d_20231225080414.jpg', '3a0f5a44-a13b-3ae8-8cad-b1e69b2614b1_3a0fb216-76d9-349f-ca2a-72cd65115f4b_20231225080441.jpg', '3a0f5a44-a13b-3ba0-1c5a-cce36dfbc102_3a0fb216-76d9-149d-6b9e-9aeca820cc2d_20231225080403.jpg', '3a0f5a44-a13b-4e81-43ee-34cd314c64eb_3a0fb216-76cd-e824-740f-4606943d467d_20231225090428.jpg', '3a0f5a44-a13b-512f-8108-341614c4ed3e_3a0fb216-76cd-aa8b-4c95-f02ac94b54d8_20231225050403.jpg', '3a0f5a44-a13b-6d85-d787-9fd616ece5a5_3a0fb216-76d9-9c4e-1fce-6ea5caeae160_20231225080423.jpg', '3a0f5a44-a13b-7131-c9d1-c97292fdc1b8_3a0fb216-76cd-aa8b-4c95-f02ac94b54d8_20231225040452.jpg', '3a0f5a44-a13b-8680-b129-83f98a812630_3a0fb216-76cd-2c11-2c21-e64fe4d1ccec_20231225090407.jpg', '3a0f5a44-a13b-9c22-0cdb-1ef88fbd354b_3a0fb216-76cd-381f-4bee-fe16451c5bf6_20231225090451.jpg', '3a0f5a44-a13b-a8e6-03b3-d3c88e3292c7_3a0fb216-76ce-4ed2-2287-6542e89f5eb4_20231225030427.jpg', '3a0f5a44-a13b-b050-6a53-095e5bb3b392_3a0fb216-76d7-e192-15be-5e7718b8eced_20231225080443.jpg', '3a0f5a44-a13b-bc14-31d1-3edcbf856018_3a0fb216-76cd-8c02-86e9-dd0c5b430b87_20231225080446.jpg', '3a0f5a44-a13b-bc5a-7e9c-2932fe3bb57f_3a0fb216-76d9-349f-ca2a-72cd65115f4b_20231225080452.jpg', '3a0f5a44-a13b-c68e-1038-d6e6800716d0_3a0fb216-76cd-2c11-2c21-e64fe4d1ccec_20231225080455.jpg', '3a0f5a44-a13b-e2f6-5670-bbda4f0f72eb_3a0fb216-76d7-e192-15be-5e7718b8eced_20231225080408.jpg', '3a0f5a44-a13b-e6ff-8941-9fa869a6ed53_3a0fb216-76cd-e650-cbdc-939260d52add_20231225090412.jpg', '3a0f5a44-a13b-ede6-8763-6fe52ced2337_3a0fb216-76d9-9c4e-1fce-6ea5caeae160_20231225080434.jpg', '3a0f5abf-3643-4933-e894-84e2f91dc390_3a0fb216-76d1-19fe-948f-3e929b4f7cd6_20231225080443.jpg', '3a0f5abf-3643-6006-797f-b4aa26fe9a24_3a0fb216-76d2-2625-f8b5-4ccbfcac02dc_20231225080434.jpg']
len(ids_error): 51