143 lines
5.4 KiB
Python
143 lines
5.4 KiB
Python
|
import pandas as pd
|
|||
|
|
import sqlalchemy
|
|||
|
|
from sqlalchemy import create_engine
|
|||
|
|
import configparser
|
|||
|
|
import os
|
|||
|
|
from urllib.parse import quote_plus
|
|||
|
|
|
|||
|
|
from datetime import datetime
|
|||
|
|
|
|||
|
|
def fix_date(date_str):
|
|||
|
|
try:
|
|||
|
|
# 处理只有年月的情况（如"2018-07"）
|
|||
|
|
if isinstance(date_str, str) and len(date_str) == 7 and date_str[4] == '-':
|
|||
|
|
return datetime.strptime(date_str + '-01', '%Y-%m-%d')
|
|||
|
|
return date_str
|
|||
|
|
except:
|
|||
|
|
return None # 对于无法解析的日期返回NULL
|
|||
|
|
|
|||
|
|
def get_db_connection_string(config, section):
|
|||
|
|
"""从配置文件中构建数据库连接字符串"""
|
|||
|
|
return f"mysql+pymysql://{config[section]['user']}:{quote_plus(config[section]['password'])}@" \
|
|||
|
|
f"{config[section]['host']}:{config[section]['port']}/{config[section]['database']}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def transform_and_load_ma_type(config_file_path):
|
|||
|
|
"""
|
|||
|
|
从源数据库提取ma_type数据，转换后加载到目标数据库ma_type
|
|||
|
|
:param config_file_path: 配置文件路径
|
|||
|
|
"""
|
|||
|
|
# 读取配置文件
|
|||
|
|
if not os.path.exists(config_file_path):
|
|||
|
|
raise FileNotFoundError(f"配置文件不存在: {config_file_path}")
|
|||
|
|
|
|||
|
|
config = configparser.ConfigParser()
|
|||
|
|
config.read(config_file_path)
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 获取数据库连接
|
|||
|
|
source_conn_str = get_db_connection_string(config, 'source_db')
|
|||
|
|
target_conn_str = get_db_connection_string(config, 'target_db')
|
|||
|
|
source_engine = create_engine(source_conn_str)
|
|||
|
|
target_engine = create_engine(target_conn_str)
|
|||
|
|
|
|||
|
|
# 从源表读取数据（过滤COMPANY_ID=1且IS_ACTIVE=1的记录）
|
|||
|
|
print("正在从源表ma_type读取数据...")
|
|||
|
|
source_query = """
|
|||
|
|
SELECT ID, \
|
|||
|
|
PARENT_ID, \
|
|||
|
|
NAME, \
|
|||
|
|
LEVEL, \
|
|||
|
|
UNIT, \
|
|||
|
|
NUM, \
|
|||
|
|
nullif(BUY_PRICE,0) as BUY_PRICE, \
|
|||
|
|
nullif(LEASE_PRICE,0) as LEASE_PRICE, \
|
|||
|
|
IS_COUNT, \
|
|||
|
|
RATED_LOAD, \
|
|||
|
|
TEST_LOAD, \
|
|||
|
|
HOLDING_TIME, \
|
|||
|
|
manage_type,
|
|||
|
|
nullif(IS_TEST,0) as IS_TEST,
|
|||
|
|
nullif(TIME,0) as TIME
|
|||
|
|
FROM ma_type
|
|||
|
|
WHERE COMPANY_ID = 1 AND IS_ACTIVE = 1 \
|
|||
|
|
"""
|
|||
|
|
source_df = pd.read_sql(source_query, source_engine)
|
|||
|
|
|
|||
|
|
if source_df.empty:
|
|||
|
|
print("没有符合条件的数据需要转换")
|
|||
|
|
return
|
|||
|
|
print(f"读取到{len(source_df)}条待转换数据")
|
|||
|
|
|
|||
|
|
# 数据转换（直接字段复制）
|
|||
|
|
print("正在进行数据转换...")
|
|||
|
|
target_df = pd.DataFrame()
|
|||
|
|
|
|||
|
|
# 字段映射（源字段 → 目标字段）
|
|||
|
|
field_mapping = {
|
|||
|
|
'ID': 'type_id',
|
|||
|
|
'PARENT_ID': 'parent_id',
|
|||
|
|
'NAME': 'type_name',
|
|||
|
|
'LEVEL': 'level',
|
|||
|
|
'UNIT': 'unit_name',
|
|||
|
|
'NUM': 'storage_num',
|
|||
|
|
'BUY_PRICE': 'buy_price',
|
|||
|
|
'LEASE_PRICE': 'lease_price',
|
|||
|
|
'IS_COUNT': 'manage_type',
|
|||
|
|
'RATED_LOAD': 'rated_load',
|
|||
|
|
'TEST_LOAD': 'test_load',
|
|||
|
|
'HOLDING_TIME': 'holding_time',
|
|||
|
|
'manage_type': 'is_enter',
|
|||
|
|
'IS_TEST': 'is_test',
|
|||
|
|
'TIME': 'create_time'
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for source_field, target_field in field_mapping.items():
|
|||
|
|
target_df[target_field] = source_df[source_field]
|
|||
|
|
# 应用日期修正
|
|||
|
|
target_df['create_time'] = source_df['TIME'].apply(fix_date)
|
|||
|
|
# 写入目标表
|
|||
|
|
print("正在写入目标表ma_type...")
|
|||
|
|
target_df.to_sql(
|
|||
|
|
'ma_type',
|
|||
|
|
target_engine,
|
|||
|
|
if_exists='append',
|
|||
|
|
index=False,
|
|||
|
|
dtype={
|
|||
|
|
'type_id': sqlalchemy.types.INTEGER(),
|
|||
|
|
'parent_id': sqlalchemy.types.INTEGER(),
|
|||
|
|
'type_name': sqlalchemy.types.VARCHAR(length=100),
|
|||
|
|
'level': sqlalchemy.types.SmallInteger(),
|
|||
|
|
'unit_name': sqlalchemy.types.VARCHAR(length=20),
|
|||
|
|
'storage_num': sqlalchemy.types.VARCHAR(length=50),
|
|||
|
|
'buy_price': sqlalchemy.types.DECIMAL(10, 2),
|
|||
|
|
'lease_price': sqlalchemy.types.DECIMAL(10, 2),
|
|||
|
|
'manage_type': sqlalchemy.types.SmallInteger(),
|
|||
|
|
'rated_load': sqlalchemy.types.DECIMAL(10, 2),
|
|||
|
|
'test_load': sqlalchemy.types.DECIMAL(10, 2),
|
|||
|
|
'holding_time': sqlalchemy.types.INTEGER(),
|
|||
|
|
'is_enter': sqlalchemy.types.SmallInteger(),
|
|||
|
|
'is_test': sqlalchemy.types.SmallInteger(),
|
|||
|
|
'create_time': sqlalchemy.types.DateTime()
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
print(f"成功写入{len(target_df)}条数据")
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"处理过程中发生错误: {str(e)}")
|
|||
|
|
raise
|
|||
|
|
finally:
|
|||
|
|
if 'source_engine' in locals():
|
|||
|
|
source_engine.dispose()
|
|||
|
|
if 'target_engine' in locals():
|
|||
|
|
target_engine.dispose()
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
# 配置文件路径（根据实际情况修改）
|
|||
|
|
config_file = "config.ini"
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
transform_and_load_ma_type(config_file)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"程序执行失败: {str(e)}")
|