292 lines
9.9 KiB
Python
292 lines
9.9 KiB
Python
import configparser
|
||
import pandas as pd
|
||
from sqlalchemy import create_engine
|
||
from urllib.parse import quote_plus
|
||
|
||
# 读取配置文件
|
||
config = configparser.ConfigParser()
|
||
config.read('config.ini')
|
||
|
||
# 获取数据库连接配置
|
||
source_config = {
|
||
'host': config.get('source_db', 'host'),
|
||
'user': config.get('source_db', 'user'),
|
||
'password': config.get('source_db', 'password'),
|
||
'database': config.get('source_db', 'database'),
|
||
'port': config.getint('source_db', 'port')
|
||
}
|
||
|
||
target_config = {
|
||
'host': config.get('target_db', 'host'),
|
||
'user': config.get('target_db', 'user'),
|
||
'password': config.get('target_db', 'password'),
|
||
'database': config.get('target_db', 'database'),
|
||
'port': config.getint('target_db', 'port')
|
||
}
|
||
|
||
# 创建数据库引擎（修正1：增加charset参数避免中文乱码）
|
||
source_engine = create_engine(
|
||
f"mysql+pymysql://{source_config['user']}:{quote_plus(source_config['password'])}@{source_config['host']}:{source_config['port']}/{source_config['database']}?charset=utf8mb4"
|
||
)
|
||
target_engine = create_engine(
|
||
f"mysql+pymysql://{target_config['user']}:{quote_plus(target_config['password'])}@{target_config['host']}:{target_config['port']}/{target_config['database']}?charset=utf8mb4"
|
||
)
|
||
|
||
# 定义type映射关系
|
||
type_mapping = {
|
||
6: 'sgd', # 独立施工队
|
||
7: 'xmb', # 项目部
|
||
122: 'fbs', # 分包商
|
||
126: 'hq' # 后勤
|
||
}
|
||
|
||
|
||
def get_agreement_id(unit_id, project_id, type_code):
|
||
"""根据组合条件查询agreement_id（使用参数化查询）"""
|
||
try:
|
||
# 修正2：添加参数有效性检查
|
||
if pd.isna(unit_id) or pd.isna(project_id) or pd.isna(type_code):
|
||
return None
|
||
|
||
sql = """
|
||
SELECT (bpr.ID + 500000) as agreement_id
|
||
FROM bm_project_relation bpr
|
||
WHERE bpr.UNIT_ID = %s
|
||
AND bpr.project_id = %s
|
||
AND bpr.type = %s
|
||
LIMIT 1
|
||
"""
|
||
params = (int(unit_id), int(project_id), str(type_code))
|
||
result = pd.read_sql(sql, source_engine, params=params)
|
||
return result['agreement_id'].iloc[0] if not result.empty else None
|
||
except Exception as e:
|
||
print(f"查询agreement_id出错: unit_id={unit_id}, project_id={project_id}, type={type_code}, 错误: {str(e)}")
|
||
return None
|
||
|
||
|
||
def process_tm_task():
|
||
"""处理维修任务主表"""
|
||
try:
|
||
sql = """
|
||
SELECT
|
||
(be.ID + 500000) as task_id,
|
||
be.`CODE`,
|
||
4 as task_type,
|
||
pu.`NAME` as create_by,
|
||
be.CREATE_TIME,
|
||
IF(be.`STATUS` = 56, 1, 0) as task_status
|
||
FROM
|
||
bpm_example be
|
||
LEFT JOIN sys_data_dict sdd on be.`STATUS` = sdd.ID
|
||
LEFT JOIN pm_user pu on be.CREATOR = pu.ID
|
||
WHERE be.DEFINITION_ID = 7
|
||
"""
|
||
df = pd.read_sql(sql, source_engine)
|
||
|
||
# 修正3：确保datetime类型正确
|
||
if 'CREATE_TIME' in df.columns:
|
||
df['CREATE_TIME'] = pd.to_datetime(df['CREATE_TIME'])
|
||
|
||
df.to_sql('tm_task', target_engine, if_exists='append', index=False)
|
||
print(f"成功导入 {len(df)} 条维修任务主表数据")
|
||
return True
|
||
except Exception as e:
|
||
print(f"处理维修任务主表时出错: {str(e)}")
|
||
return False
|
||
|
||
|
||
def process_tm_task_agreement():
|
||
"""处理维修任务关联表"""
|
||
try:
|
||
# 第一步查询基础数据
|
||
base_sql = """
|
||
SELECT
|
||
(be.ID + 500000) as task_id,
|
||
be.DEPT_ID,
|
||
be.SUB_ID,
|
||
be.TEAM_ID,
|
||
be.REAR_ID,
|
||
be.TYPE,
|
||
be.PROJECT_ID
|
||
FROM bpm_example be
|
||
WHERE be.DEFINITION_ID = 7
|
||
"""
|
||
base_df = pd.read_sql(base_sql, source_engine)
|
||
|
||
# 修正4：更安全的空值处理
|
||
base_df = base_df.fillna({
|
||
'DEPT_ID': 0, 'SUB_ID': 0,
|
||
'TEAM_ID': 0, 'REAR_ID': 0,
|
||
'PROJECT_ID': 0, 'TYPE': 0
|
||
})
|
||
|
||
# 修正5：确保所有ID为整数
|
||
for col in ['DEPT_ID', 'SUB_ID', 'TEAM_ID', 'REAR_ID', 'PROJECT_ID', 'TYPE']:
|
||
base_df[col] = pd.to_numeric(base_df[col], errors='coerce').fillna(0).astype(int)
|
||
|
||
results = []
|
||
skipped_records = 0
|
||
|
||
for _, row in base_df.iterrows():
|
||
try:
|
||
# 确定unit_id和type_code
|
||
unit_id = None
|
||
type_code = None
|
||
|
||
if row['TYPE'] == 6 and row['TEAM_ID'] > 0: # 施工队
|
||
unit_id = row['TEAM_ID'] + 5000
|
||
type_code = 'sgd'
|
||
elif row['TYPE'] == 7 and row['DEPT_ID'] > 0: # 项目部
|
||
unit_id = row['DEPT_ID'] + 4000
|
||
type_code = 'xmb'
|
||
elif row['TYPE'] == 122 and row['SUB_ID'] > 0: # 分包商
|
||
unit_id = row['SUB_ID'] + 6000
|
||
type_code = 'fbs'
|
||
elif row['TYPE'] == 126 and row['REAR_ID'] > 0: # 后勤
|
||
unit_id = row['REAR_ID'] + 6000
|
||
type_code = 'hq'
|
||
|
||
if not unit_id or row['PROJECT_ID'] <= 0:
|
||
skipped_records += 1
|
||
continue
|
||
|
||
agreement_id = get_agreement_id(unit_id, row['PROJECT_ID'], type_code)
|
||
if agreement_id:
|
||
results.append({
|
||
'task_id': int(row['task_id']),
|
||
'agreement_id': int(agreement_id)
|
||
})
|
||
except Exception as e:
|
||
print(f"处理记录时出错: {str(e)}，记录ID: {row.get('task_id', '未知')}")
|
||
skipped_records += 1
|
||
continue
|
||
|
||
if results:
|
||
result_df = pd.DataFrame(results)
|
||
result_df.to_sql('tm_task_agreement', target_engine,
|
||
if_exists='append', index=False)
|
||
print(f"成功导入 {len(result_df)} 条维修任务关联数据，跳过 {skipped_records} 条无效记录")
|
||
return True
|
||
except Exception as e:
|
||
print(f"处理维修任务关联表时出错: {str(e)}")
|
||
return False
|
||
|
||
|
||
def safe_convert_to_int(series):
|
||
"""
|
||
安全转换为整数，处理以下情况：
|
||
- 空字符串('')
|
||
- 浮点字符串('1.0')
|
||
- 空值(None/NaN)
|
||
- 常规整数字符串('123')
|
||
"""
|
||
# 先转换为字符串，处理各种输入类型
|
||
str_series = series.astype(str)
|
||
# 替换空字符串为'0'
|
||
str_series = str_series.replace({'': '0'})
|
||
# 转换为float再转为int
|
||
return pd.to_numeric(str_series, errors='coerce').fillna(0).astype(int)
|
||
|
||
|
||
def process_repair_apply_details():
|
||
"""处理维修申请明细"""
|
||
try:
|
||
sql = """
|
||
SELECT
|
||
(brf.EXAMPLE_ID + 500000) as task_id,
|
||
(brf.MATYPE_ID + 6000) as type_id,
|
||
(brf.MA_ID + 70000) as ma_id,
|
||
brf.BACK_NUM as repair_num,
|
||
brf.REPAIR_NUM as repaired_num,
|
||
brf.SCRAP_NUM as scrap_num,
|
||
IF(be.`STATUS` = 56, 1, 0) as status,
|
||
pu.`NAME` as create_by,
|
||
be.PARENT_ID as back_id,
|
||
IF(be.`STATUS` = 56, 0, 1) as is_ds
|
||
FROM
|
||
bpm_repair_form brf
|
||
LEFT JOIN bpm_example be on brf.EXAMPLE_ID = be.ID
|
||
LEFT JOIN pm_user pu on be.CREATOR = pu.ID
|
||
WHERE be.DEFINITION_ID = 7
|
||
"""
|
||
df = pd.read_sql(sql, source_engine)
|
||
|
||
# 转换所有数值列
|
||
num_cols = ['repair_num', 'repaired_num', 'scrap_num', 'status', 'is_ds', 'back_id']
|
||
for col in num_cols:
|
||
if col in df.columns:
|
||
df[col] = safe_convert_to_int(df[col])
|
||
|
||
# 记录转换前后的样本值
|
||
print("维修申请明细表转换样例：")
|
||
for col in num_cols:
|
||
if col in df.columns and len(df) > 0:
|
||
sample = df[col].iloc[0]
|
||
print(f"{col}: {sample} (类型: {type(sample)})")
|
||
|
||
df.to_sql('repair_apply_details', target_engine,
|
||
if_exists='append', index=False)
|
||
print(f"成功导入 {len(df)} 条维修申请明细")
|
||
return True
|
||
except Exception as e:
|
||
print(f"处理维修申请明细时出错: {str(e)}")
|
||
return False
|
||
|
||
|
||
def process_repair_apply_record():
|
||
"""处理维修申请记录"""
|
||
try:
|
||
sql = """
|
||
SELECT
|
||
(brf.EXAMPLE_ID + 500000) as task_id,
|
||
(brf.MATYPE_ID + 6000) as type_id,
|
||
(brf.MA_ID + 70000) as ma_id,
|
||
brf.REPAIR_NUM as repair_num,
|
||
brf.SCRAP_NUM as scrap_num,
|
||
IF(be.`STATUS` = 56, 1, 0) as status,
|
||
pu.`NAME` as create_by
|
||
FROM
|
||
bpm_repair_form brf
|
||
LEFT JOIN bpm_example be on brf.EXAMPLE_ID = be.ID
|
||
LEFT JOIN pm_user pu on be.CREATOR = pu.ID
|
||
WHERE be.`STATUS` = 56 AND be.DEFINITION_ID = 7
|
||
"""
|
||
df = pd.read_sql(sql, source_engine)
|
||
|
||
# 转换所有数值列
|
||
num_cols = ['repair_num', 'scrap_num', 'status']
|
||
for col in num_cols:
|
||
if col in df.columns:
|
||
df[col] = safe_convert_to_int(df[col])
|
||
|
||
# 记录转换前后的样本值
|
||
print("维修申请记录表转换样例：")
|
||
for col in num_cols:
|
||
if col in df.columns and len(df) > 0:
|
||
sample = df[col].iloc[0]
|
||
print(f"{col}: {sample} (类型: {type(sample)})")
|
||
|
||
df.to_sql('repair_apply_record', target_engine,
|
||
if_exists='append', index=False)
|
||
print(f"成功导入 {len(df)} 条维修申请记录")
|
||
return True
|
||
except Exception as e:
|
||
print(f"处理维修申请记录时出错: {str(e)}")
|
||
return False
|
||
|
||
|
||
if __name__ == "__main__":
|
||
# 执行所有转换流程
|
||
processes = [
|
||
process_tm_task,
|
||
process_tm_task_agreement,
|
||
process_repair_apply_details,
|
||
process_repair_apply_record
|
||
]
|
||
|
||
success = all([p() for p in processes])
|
||
|
||
if success:
|
||
print("所有维修相关表转换完成！")
|
||
else:
|
||
print("!!! 部分转换失败，请检查错误日志 !!!") |