129 lines
4.9 KiB
Python
129 lines
4.9 KiB
Python
|
import pandas as pd
|
|||
|
|
import sqlalchemy
|
|||
|
|
from sqlalchemy import create_engine
|
|||
|
|
import configparser
|
|||
|
|
import os
|
|||
|
|
from urllib.parse import quote_plus
|
|||
|
|
|
|||
|
|
def get_db_connection_string(config, section):
|
|||
|
|
"""从配置文件中构建数据库连接字符串"""
|
|||
|
|
return f"mysql+pymysql://{config[section]['user']}:{quote_plus(config[section]['password'])}@" \
|
|||
|
|
f"{config[section]['host']}:{config[section]['port']}/{config[section]['database']}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def transform_and_load_agreement(config_file_path):
|
|||
|
|
"""
|
|||
|
|
从源数据库提取ba_ma_agreement数据，转换后加载到目标数据库bm_agreement_info
|
|||
|
|
:param config_file_path: 配置文件路径
|
|||
|
|
"""
|
|||
|
|
# 读取配置文件
|
|||
|
|
if not os.path.exists(config_file_path):
|
|||
|
|
raise FileNotFoundError(f"配置文件不存在: {config_file_path}")
|
|||
|
|
|
|||
|
|
config = configparser.ConfigParser()
|
|||
|
|
config.read(config_file_path)
|
|||
|
|
|
|||
|
|
# 定义替换规则
|
|||
|
|
is_slt_mapping = {
|
|||
|
|
0: 1, # 已结算 → 1
|
|||
|
|
1: 0 # 未结算 → 0
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 获取数据库连接
|
|||
|
|
source_conn_str = get_db_connection_string(config, 'source_db')
|
|||
|
|
target_conn_str = get_db_connection_string(config, 'target_db')
|
|||
|
|
source_engine = create_engine(source_conn_str)
|
|||
|
|
target_engine = create_engine(target_conn_str)
|
|||
|
|
|
|||
|
|
# 从源表读取数据（过滤COMPANY_ID=1的记录）
|
|||
|
|
print("正在从源表ba_ma_agreement读取数据...")
|
|||
|
|
source_query = """
|
|||
|
|
SELECT ID, \
|
|||
|
|
CODE, \
|
|||
|
|
SIGN_DATE, \
|
|||
|
|
LEASE_COMPANY, \
|
|||
|
|
PROJECT, \
|
|||
|
|
START_TIME,
|
|||
|
|
AUTHORIZING_PERSON, \
|
|||
|
|
AUTHORIZING_PHONE, \
|
|||
|
|
CONTRACT_NUMBER,
|
|||
|
|
IS_BALANCE, \
|
|||
|
|
is_push, CREATE_TIME
|
|||
|
|
FROM ba_ma_agreement
|
|||
|
|
WHERE COMPANY_ID = 1 \
|
|||
|
|
"""
|
|||
|
|
source_df = pd.read_sql(source_query, source_engine)
|
|||
|
|
|
|||
|
|
if source_df.empty:
|
|||
|
|
print("没有符合条件的数据需要转换")
|
|||
|
|
return
|
|||
|
|
print(f"读取到{len(source_df)}条待转换数据")
|
|||
|
|
|
|||
|
|
# 数据转换
|
|||
|
|
print("正在进行数据转换...")
|
|||
|
|
target_df = pd.DataFrame()
|
|||
|
|
|
|||
|
|
# 直接复制字段
|
|||
|
|
target_df['agreement_id'] = source_df['ID']
|
|||
|
|
target_df['agreement_code'] = source_df['CODE']
|
|||
|
|
target_df['sign_time'] = source_df['SIGN_DATE']
|
|||
|
|
target_df['unit_id'] = source_df['LEASE_COMPANY']
|
|||
|
|
target_df['project_id'] = source_df['PROJECT']
|
|||
|
|
target_df['plan_start_time'] = source_df['START_TIME']
|
|||
|
|
target_df['auth_person'] = source_df['AUTHORIZING_PERSON']
|
|||
|
|
target_df['phone'] = source_df['AUTHORIZING_PHONE']
|
|||
|
|
target_df['contract_code'] = source_df['CONTRACT_NUMBER']
|
|||
|
|
target_df['is_push'] = source_df['is_push']
|
|||
|
|
target_df['create_time'] = source_df['CREATE_TIME']
|
|||
|
|
|
|||
|
|
# 替换字段（IS_BALANCE → is_slt）
|
|||
|
|
target_df['is_slt'] = source_df['IS_BALANCE'].map(is_slt_mapping)
|
|||
|
|
|
|||
|
|
# 检查未映射的值
|
|||
|
|
if target_df['is_slt'].isna().any():
|
|||
|
|
unmapped = source_df[target_df['is_slt'].isna()]['IS_BALANCE'].unique()
|
|||
|
|
print(f"警告: 发现未映射的IS_BALANCE值: {unmapped}")
|
|||
|
|
|
|||
|
|
# 写入目标表
|
|||
|
|
print("正在写入目标表bm_agreement_info...")
|
|||
|
|
target_df.to_sql(
|
|||
|
|
'bm_agreement_info',
|
|||
|
|
target_engine,
|
|||
|
|
if_exists='append',
|
|||
|
|
index=False,
|
|||
|
|
dtype={
|
|||
|
|
'agreement_id': sqlalchemy.types.INTEGER(),
|
|||
|
|
'agreement_code': sqlalchemy.types.VARCHAR(length=50),
|
|||
|
|
'sign_time': sqlalchemy.types.DateTime(),
|
|||
|
|
'unit_id': sqlalchemy.types.INTEGER(),
|
|||
|
|
'project_id': sqlalchemy.types.INTEGER(),
|
|||
|
|
'plan_start_time': sqlalchemy.types.DateTime(),
|
|||
|
|
'auth_person': sqlalchemy.types.VARCHAR(length=50),
|
|||
|
|
'phone': sqlalchemy.types.VARCHAR(length=20),
|
|||
|
|
'contract_code': sqlalchemy.types.VARCHAR(length=50),
|
|||
|
|
'is_slt': sqlalchemy.types.SmallInteger(),
|
|||
|
|
'is_push': sqlalchemy.types.SmallInteger(),
|
|||
|
|
'create_time': sqlalchemy.types.DateTime()
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
print(f"成功写入{len(target_df)}条数据")
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"处理过程中发生错误: {str(e)}")
|
|||
|
|
raise
|
|||
|
|
finally:
|
|||
|
|
if 'source_engine' in locals():
|
|||
|
|
source_engine.dispose()
|
|||
|
|
if 'target_engine' in locals():
|
|||
|
|
target_engine.dispose()
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
# 配置文件路径（根据实际情况修改）
|
|||
|
|
config_file = "config.ini"
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
transform_and_load_agreement(config_file)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"程序执行失败: {str(e)}")
|