liurui/PriceForecast
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

聚烯烃调试&埃森哲代码

Browse Source
This commit is contained in:
workpc 2025-05-26 14:53:42 +08:00
parent 448f2989d4
commit 98b587168c
38 changed files with 5651 additions and 908 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.idea/.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

12
.idea/PriceForecast-svn.iml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.10 (predict)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
</module>

6
.idea/inspectionProfiles/profiles_settings.xml Normal file
View File

@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

4
.idea/misc.xml Normal file
View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (predict)" project-jdk-type="Python SDK" />
</project>

8
.idea/modules.xml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/PriceForecast-svn.iml" filepath="$PROJECT_DIR$/.idea/PriceForecast-svn.iml" />
</modules>
</component>
</project>

6
.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

BIN
aisenzhecode/沥青/定性模型数据项12-11.xlsx
View File

Binary file not shown.

BIN
aisenzhecode/沥青/日度价格预测_最佳模型.pkl
View File

Binary file not shown.

BIN
aisenzhecode/沥青/日度价格预测_最佳模型.pkl.r87044 Normal file
View File

Binary file not shown.

BIN
aisenzhecode/沥青/日度价格预测_最佳模型.pkl.r87201 Normal file
View File

Binary file not shown.

469
aisenzhecode/沥青/沥青定性模型每日推送-ytj.ipynb
View File

@ -1169,7 +1169,474 @@
"name": "stdout",
"output_type": "stream",
"text": [
"运行中...\n"
"运行中...\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"1428 2025-05-06 3650 29.7819 1 -798.61 6000 90799.9 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"1428 2934.18 1 4876.81 7364.99 \n",
"日期存在，即将更新\n",
"新数据 [3650.0, '', '', -798.61, '', 90799.9179153, '', 2752.8654, '', '', 7364.987]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"1428 2025-05-06 3650 29.7819 1 -798.61 6000 90799.9 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"1428 2934.18 1 4876.81 7364.99 \n",
"20250507\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"1427 2025-05-05 3650 29.3624 1 -798.61 6000 90799.9 3630 \n",
"1428 2025-05-06 3650 32.2987 1 -454.19 6000 94300.4 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"1427 2752.87 1 4876.81 7364.99 \n",
"1428 2934.18 1 4876.81 6793.88 \n",
"前一天的 2752.8654 <class 'float'>\n",
"现在的 2934.179 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1427 2025-05-05 3650 29.3624 1 -798.61 6000 90799.9 3630 \n",
"1 1428 2025-05-06 3650 32.2987 1 -454.19 6000 94300.4 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 2752.87 1 4876.81 7364.99 \n",
"1 2934.18 1 4876.81 6793.88 \n",
"昨日计划提货偏差改之前 -454.19\n",
"昨日计划提货偏差改之后 1917.0684\n",
"**************************************************预测结果： 3689.0\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"1429 2025-05-07 3650 32.2987 1 -454.19 6000 94300.4 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"1429 3036.84 1 4876.81 6793.88 \n",
"日期存在，即将更新\n",
"新数据 [3650.0, 32.2987, '', -454.19, '', 94300.352488, '', 2934.179, '', '', 6793.8778]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"1429 2025-05-07 3650 32.2987 1 -454.19 6000 94300.4 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"1429 3036.84 1 4876.81 6793.88 \n",
"20250508\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"1428 2025-05-06 3650 32.2987 1 -454.19 6000 94300.4 3630 \n",
"1429 2025-05-07 3600 33.9765 1 693.59 6000 97550.8 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"1428 2934.18 1 4876.81 6793.88 \n",
"1429 3036.84 1 4876.81 7237.56 \n",
"前一天的 2934.179 <class 'float'>\n",
"现在的 3036.8392 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1428 2025-05-06 3650 32.2987 1 -454.19 6000 94300.4 3630 \n",
"1 1429 2025-05-07 3600 33.9765 1 693.59 6000 97550.8 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 2934.18 1 4876.81 6793.88 \n",
"1 3036.84 1 4876.81 7237.56 \n",
"昨日计划提货偏差改之前 693.59\n",
"昨日计划提货偏差改之后 2360.7506000000003\n",
"**************************************************预测结果： 3673.89\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"1430 2025-05-08 3600 33.9765 1 693.59 6000 97550.8 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"1430 2940.67 1 4876.81 7237.56 \n",
"日期存在，即将更新\n",
"新数据 [3600.0, 33.9765, '', 693.59, '', 97550.7881218, '', 3036.8392, '', '', 7237.56]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"1430 2025-05-08 3600 33.9765 1 693.59 6000 97550.8 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"1430 2940.67 1 4876.81 7237.56 \n",
"20250509\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"1429 2025-05-07 3600 33.9765 1 693.59 6000 97550.8 3630 \n",
"1430 2025-05-08 3600 34.396 1 329.51 6000 100086 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"1429 3036.84 1 4876.81 7237.56 \n",
"1430 2940.67 1 4876.81 7711.98 \n",
"前一天的 3036.8392 <class 'float'>\n",
"现在的 2940.6669 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1429 2025-05-07 3600 33.9765 1 693.59 6000 97550.8 3630 \n",
"1 1430 2025-05-08 3600 34.396 1 329.51 6000 100086 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 3036.84 1 4876.81 7237.56 \n",
"1 2940.67 1 4876.81 7711.98 \n",
"昨日计划提货偏差改之前 329.51\n",
"昨日计划提货偏差改之后 2835.1686\n",
"**************************************************预测结果： 3604.08\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1431 2025-05-09 3600 34.396 1 329.51 6000 100086 3630 3077.15 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1431 1 4876.81 7711.98 \n",
"日期存在，即将更新\n",
"新数据 [3600.0, 34.396, '', 329.51, '', 100085.7741619, '', 2940.6669, '', '', 7711.978]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1431 2025-05-09 3600 34.396 1 329.51 6000 100086 3630 3077.15 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1431 1 4876.81 7711.98 \n",
"20250510\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1430 2025-05-08 3600 34.396 1 329.51 6000 100086 3630 2940.67 \n",
"1431 2025-05-09 3620 36.9128 1 -1636.9 6000 101412 3630 3077.15 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1430 1 4876.81 7711.98 \n",
"1431 1 4876.81 7246.98 \n",
"前一天的 2940.6669 <class 'float'>\n",
"现在的 3077.1512 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1430 2025-05-08 3600 34.396 1 329.51 6000 100086 3630 \n",
"1 1431 2025-05-09 3620 36.9128 1 -1636.9 6000 101412 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 2940.67 1 4876.81 7711.98 \n",
"1 3077.15 1 4876.81 7246.98 \n",
"昨日计划提货偏差改之前 -1636.9\n",
"昨日计划提货偏差改之后 2370.1705999999995\n",
"**************************************************预测结果： 3630.55\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1432 2025-05-10 3620 36.9128 1 -1636.9 6000 101412 3630 3152.87 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1432 1 4876.81 7246.98 \n",
"日期存在，即将更新\n",
"新数据 [3620.0, 36.9128, '', -1636.9, '', 101412.2754832, '', 3077.1512, '', '', 7246.98]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1432 2025-05-10 3620 36.9128 1 -1636.9 6000 101412 3630 3152.87 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1432 1 4876.81 7246.98 \n",
"20250511\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1431 2025-05-09 3620 36.9128 1 -1636.9 6000 101412 3630 3077.15 \n",
"1432 2025-05-10 3620 36.9128 1 3087.72 6000 104559 3630 3152.87 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1431 1 4876.81 7246.98 \n",
"1432 1 4876.81 7347.37 \n",
"前一天的 3077.1512 <class 'float'>\n",
"现在的 3152.8717 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1431 2025-05-09 3620 36.9128 1 -1636.9 6000 101412 3630 \n",
"1 1432 2025-05-10 3620 36.9128 1 3087.72 6000 104559 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 3077.15 1 4876.81 7246.98 \n",
"1 3152.87 1 4876.81 7347.37 \n",
"昨日计划提货偏差改之前 3087.72\n",
"昨日计划提货偏差改之后 2470.5621\n",
"**************************************************预测结果： 3623.41\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1433 2025-05-11 3620 36.9128 1 3087.72 6000 104559 3630 3151.4 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1433 1 4876.81 7347.37 \n",
"日期存在，即将更新\n",
"新数据 [3620.0, '', '', 3087.72, '', 104559.3953618, '', 3152.8717, '', '', 7347.3715]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1433 2025-05-11 3620 36.9128 1 3087.72 6000 104559 3630 3151.4 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1433 1 4876.81 7347.37 \n",
"20250512\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1432 2025-05-10 3620 36.9128 1 3087.72 6000 104559 3630 3152.87 \n",
"1433 2025-05-11 3620 36.9128 1 2942.36 6000 106829 3630 3151.4 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1432 1 4876.81 7347.37 \n",
"1433 1 4876.81 7365.94 \n",
"前一天的 3152.8717 <class 'float'>\n",
"现在的 3151.3966 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1432 2025-05-10 3620 36.9128 1 3087.72 6000 104559 3630 \n",
"1 1433 2025-05-11 3620 36.9128 1 2942.36 6000 106829 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 3152.87 1 4876.81 7347.37 \n",
"1 3151.4 1 4876.81 7365.94 \n",
"昨日计划提货偏差改之前 2942.36\n",
"昨日计划提货偏差改之后 2489.1256000000003\n",
"**************************************************预测结果： 3623.44\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1434 2025-05-12 3620 36.9128 1 2942.36 6000 106829 3630 3171.09 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1434 1 4876.81 7365.94 \n",
"日期存在，即将更新\n",
"新数据 [3620.0, '', '', 2942.36, '', 106829.1756043, '', 3151.3966, '', '', 7365.935]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1434 2025-05-12 3620 36.9128 1 2942.36 6000 106829 3630 3171.09 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1434 1 4876.81 7365.94 \n",
"20250513\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1433 2025-05-11 3620 36.9128 1 2942.36 6000 106829 3630 3151.4 \n",
"1434 2025-05-12 3620 34.396 1 251.7 6000 106775 3630 3171.09 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1433 1 4876.81 7365.94 \n",
"1434 1 4876.81 7267.18 \n",
"前一天的 3151.3966 <class 'float'>\n",
"现在的 3171.0934 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1433 2025-05-11 3620 36.9128 1 2942.36 6000 106829 3630 \n",
"1 1434 2025-05-12 3620 34.396 1 251.7 6000 106775 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 3151.4 1 4876.81 7365.94 \n",
"1 3171.09 1 4876.81 7267.18 \n",
"昨日计划提货偏差改之前 251.7\n",
"昨日计划提货偏差改之后 2390.3720999999996\n",
"**************************************************预测结果： 3623.39\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1435 2025-05-13 3620 34.396 1 251.7 6000 106775 3630 3191.69 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1435 1 4876.81 7267.18 \n",
"日期存在，即将更新\n",
"新数据 [3620.0, 34.396, '', 251.7, '', 106775.4686698, '', 3171.0934, '', '', 7267.1815]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1435 2025-05-13 3620 34.396 1 251.7 6000 106775 3630 3191.69 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1435 1 4876.81 7267.18 \n",
"20250514\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1434 2025-05-12 3620 34.396 1 251.7 6000 106775 3630 3171.09 \n",
"1435 2025-05-13 3600 34.396 1 -256.33 6000 106220 3630 3191.69 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1434 1 4876.81 7267.18 \n",
"1435 1 4876.81 7248.69 \n",
"前一天的 3171.0934 <class 'float'>\n",
"现在的 3191.6925 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1434 2025-05-12 3620 34.396 1 251.7 6000 106775 3630 \n",
"1 1435 2025-05-13 3600 34.396 1 -256.33 6000 106220 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 3171.09 1 4876.81 7267.18 \n",
"1 3191.69 1 4876.81 7248.69 \n",
"昨日计划提货偏差改之前 -256.33\n",
"昨日计划提货偏差改之后 2371.8786\n",
"**************************************************预测结果： 3623.36\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1436 2025-05-14 3600 34.396 1 -256.33 6000 106220 3630 3318.29 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1436 1 4876.81 7248.69 \n",
"日期存在，即将更新\n",
"新数据 [3600.0, 34.396, '', -256.33, '', 106220.4490483, '', 3191.6925, '', '', 7248.688]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1436 2025-05-14 3600 34.396 1 -256.33 6000 106220 3630 3318.29 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1436 1 4876.81 7248.69 \n",
"20250515\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1435 2025-05-13 3600 34.396 1 -256.33 6000 106220 3630 3191.69 \n",
"1436 2025-05-14 3600 33.557 1 -1015.89 6000 106321 3630 3318.29 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1435 1 4876.81 7248.69 \n",
"1436 1 4876.81 7200.57 \n",
"前一天的 3191.6925 <class 'float'>\n",
"现在的 3318.2936 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1435 2025-05-13 3600 34.396 1 -256.33 6000 106220 3630 \n",
"1 1436 2025-05-14 3600 33.557 1 -1015.89 6000 106321 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 3191.69 1 4876.81 7248.69 \n",
"1 3318.29 1 4876.81 7200.57 \n",
"昨日计划提货偏差改之前 -1015.89\n",
"昨日计划提货偏差改之后 2323.7650999999996\n",
"**************************************************预测结果： 3628.64\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1437 2025-05-15 3600 33.557 1 -1015.89 6000 106321 3630 3170.14 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1437 1 4876.81 7200.57 \n",
"日期存在，即将更新\n",
"新数据 [3600.0, 33.557, '', -1015.89, '', 106321.4042938, '', 3318.2936, '', '', 7200.5745]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1437 2025-05-15 3600 33.557 1 -1015.89 6000 106321 3630 3170.14 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1437 1 4876.81 7200.57 \n",
"20250516\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1436 2025-05-14 3600 33.557 1 -1015.89 6000 106321 3630 3318.29 \n",
"1437 2025-05-15 3600 33.9765 1 1408.4 6000 105595 3630 3170.14 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1436 1 4876.81 7200.57 \n",
"1437 1 4876.81 7298.53 \n",
"前一天的 3318.2936 <class 'float'>\n",
"现在的 3170.1365 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1436 2025-05-14 3600 33.557 1 -1015.89 6000 106321 3630 \n",
"1 1437 2025-05-15 3600 33.9765 1 1408.4 6000 105595 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 3318.29 1 4876.81 7200.57 \n",
"1 3170.14 1 4876.81 7298.53 \n",
"昨日计划提货偏差改之前 1408.4\n",
"昨日计划提货偏差改之后 2421.7255999999998\n",
"**************************************************预测结果： 3573.82\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1438 2025-05-16 3620 33.9765 1 1408.4 6000 105595 3630 3164.18 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1438 1 4876.81 7298.53 \n",
"日期存在，即将更新\n",
"新数据 [3600.0, 33.9765, '', 1408.4, '', 105595.49, '', 3170.1365, '', '', 7298.535]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1438 2025-05-16 3620 33.9765 1 1408.4 6000 105595 3630 3164.18 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1438 1 4876.81 7298.53 \n",
"20250517\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1437 2025-05-15 3600 33.9765 1 1408.4 6000 105595 3630 3170.14 \n",
"1438 2025-05-16 3620 34.8154 1 -1308.63 6000 107985 3630 3164.18 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1437 1 4876.81 7298.53 \n",
"1438 1 4876.81 7407.17 \n",
"前一天的 3170.1365 <class 'float'>\n",
"现在的 3164.177 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1437 2025-05-15 3600 33.9765 1 1408.4 6000 105595 3630 \n",
"1 1438 2025-05-16 3620 34.8154 1 -1308.63 6000 107985 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 3170.14 1 4876.81 7298.53 \n",
"1 3164.18 1 4876.81 7407.17 \n",
"昨日计划提货偏差改之前 -1308.63\n",
"昨日计划提货偏差改之后 2530.3635999999997\n",
"**************************************************预测结果： 3603.59\n",
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1439 2025-05-17 3620 34.8154 1 -1308.63 6000 107985 3630 3246.69 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1439 1 4876.81 7407.17 \n",
"日期存在，即将更新\n",
"新数据 [3620.0, 34.8154, '', -1308.63, '', 107984.71, '', 3164.177, '', '', 7407.173]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1439 2025-05-17 3620 34.8154 1 -1308.63 6000 107985 3630 3246.69 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1439 1 4876.81 7407.17 \n",
"20250518\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1438 2025-05-16 3620 34.8154 1 -1308.63 6000 107985 3630 3164.18 \n",
"1439 2025-05-17 3620 34.8154 1 3334.9 6000 109984 3630 3246.69 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1438 1 4876.81 7407.17 \n",
"1439 1 4876.81 7029.96 \n",
"前一天的 3164.177 <class 'float'>\n",
"现在的 3246.6882 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1438 2025-05-16 3620 34.8154 1 -1308.63 6000 107985 3630 \n",
"1 1439 2025-05-17 3620 34.8154 1 3334.9 6000 109984 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 3164.18 1 4876.81 7407.17 \n",
"1 3246.69 1 4876.81 7029.96 \n",
"昨日计划提货偏差改之前 3334.9\n",
"昨日计划提货偏差改之后 2153.1485999999995\n",
"**************************************************预测结果： 3623.0\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"更新前一天数据\n",
"更新数据前\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1440 2025-05-18 3620 34.8154 1 3334.9 6000 109984 3630 3249.93 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1440 1 4876.81 7029.96 \n",
"日期存在，即将更新\n",
"新数据 [3620.0, '', '', 3334.9, '', 109984.19, '', 3246.6882, '', '', 7029.958]\n",
"更新数据后\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1440 2025-05-18 3620 34.8154 1 3334.9 6000 109984 3630 3249.93 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1440 1 4876.81 7029.96 \n",
"20250519\n",
" 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 即期成本 \\\n",
"1439 2025-05-17 3620 34.8154 1 3334.9 6000 109984 3630 3246.69 \n",
"1440 2025-05-18 3620 34.8154 1 793.84 6000 110418 3630 3249.93 \n",
"\n",
" 订单结构 计划产量 京博产量 \n",
"1439 1 4876.81 7029.96 \n",
"1440 1 4876.81 6533.09 \n",
"前一天的 3246.6882 <class 'float'>\n",
"现在的 3249.9268 <class 'float'>\n",
" index 日期 京博指导价 70号沥青开工率 资金因素 昨日计划提货偏差 生产情况 基质沥青库存 下游客户价格预期 \\\n",
"0 1439 2025-05-17 3620 34.8154 1 3334.9 6000 109984 3630 \n",
"1 1440 2025-05-18 3620 34.8154 1 793.84 6000 110418 3630 \n",
"\n",
" 即期成本 订单结构 计划产量 京博产量 \n",
"0 3246.69 1 4876.81 7029.96 \n",
"1 3249.93 1 4876.81 6533.09 \n",
"昨日计划提货偏差改之前 793.84\n",
"昨日计划提货偏差改之后 1656.2806\n",
"**************************************************预测结果： 3622.23\n"
]
}
],

703
aisenzhecode/沥青/沥青定量价格预测每日推送-ytj.ipynb
View File

@ -2,14 +2,56 @@
"cells": [
{
"cell_type": "code",
"execution_count": 12,
<<<<<<< .mine
"execution_count": null,
||||||| .r87044
"execution_count": 1,
=======
"execution_count": 5,
>>>>>>> .r87201
"metadata": {},
"outputs": [
{
<<<<<<< .mine
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From C:\\Users\\EDY\\AppData\\Roaming\\Python\\Python311\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
"\n"
]
},
{
||||||| .r87044
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\statsmodels\\compat\\pandas.py:49: FutureWarning:\n",
"\n",
"The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
"\n"
]
},
{
=======
>>>>>>> .r87201
"data": {
"text/html": [
" <script type=\"text/javascript\">\n",
" window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
<<<<<<< .mine
" if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (typeof require !== 'undefined') {\n",
" require.undef(\"plotly\");\n",
" requirejs.config({\n",
" paths: {\n",
" 'plotly': ['https://cdn.plot.ly/plotly-2.12.1.min']\n",
" }\n",
" });\n",
" require(['plotly'], function(Plotly) {\n",
" window._Plotly = Plotly;\n",
" });\n",
" }\n",
||||||| .r87044
" if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (typeof require !== 'undefined') {\n",
" require.undef(\"plotly\");\n",
@ -22,7 +64,11 @@
" window._Plotly = Plotly;\n",
" });\n",
" }\n",
=======
" if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
>>>>>>> .r87201
" </script>\n",
" <script type=\"module\">import \"https://cdn.plot.ly/plotly-3.0.1.min\"</script>\n",
" "
]
},
@ -33,8 +79,6 @@
"source": [
"import requests\n",
"import json\n",
"import xlrd\n",
"import xlwt\n",
"from datetime import datetime, timedelta\n",
"import time\n",
"import pandas as pd\n",
@ -151,7 +195,7 @@
"\n",
" search_data = {\n",
" \"data\": {\n",
" \"date\": get_cur_time(date)[0],\n",
" \"date\": getNow(date)[0],\n",
" \"dataItemNoList\": dataItemNoList\n",
" },\n",
" \"funcModule\": \"数据项\",\n",
@ -169,24 +213,64 @@
"\n",
"# xls文件处理\n",
"\n",
"def get_cur_time(date = ''):\n",
" if date == '':\n",
" now = datetime.now()\n",
" else:\n",
" now = date\n",
" year = now.year\n",
" month = now.month\n",
" day = now.day\n",
"\n",
" if month < 10:\n",
" month = \"0\" + str(month)\n",
" if day < 10:\n",
" day = \"0\" + str(day)\n",
" cur_time = str(year) + str(month) + str(day)\n",
" cur_time2 = str(year) + \"-\" + str(month) + \"-\" + str(day)\n",
"# cur_time = '20231007'\n",
"# cur_time2 = '2023-10-07'\n",
" return cur_time, cur_time2\n",
"\n",
"def getNow(date='', offset=0):\n",
" \"\"\"生成指定日期的两种格式字符串\n",
" Args:\n",
" date: 支持多种输入类型：\n",
" - datetime对象\n",
" - 字符串格式（支持'%Y-%m-%d'和'%Y%m%d'）\n",
" - 空字符串表示当前日期\n",
" offset: 日期偏移天数\n",
" Returns:\n",
" tuple: (紧凑日期字符串, 标准日期字符串)\n",
" \"\"\"\n",
" # 日期解析逻辑\n",
" from datetime import datetime,timedelta\n",
" if isinstance(date, datetime):\n",
" now = date\n",
" else:\n",
" now = datetime.now()\n",
" if date:\n",
" # 尝试多种日期格式解析\n",
" for fmt in ('%Y-%m-%d', '%Y%m%d', '%Y/%m/%d'):\n",
" try:\n",
" now = datetime.strptime(str(date), fmt)\n",
" break\n",
" except ValueError:\n",
" continue\n",
" else:\n",
" raise ValueError(f\"无法解析的日期格式: {date}\")\n",
"\n",
" # 应用日期偏移\n",
" now = now - timedelta(days=offset)\n",
" \n",
" # 统一格式化输出\n",
" date_str = now.strftime(\"%Y-%m-%d\")\n",
" compact_date = date_str.replace(\"-\", \"\")\n",
" return compact_date, date_str\n",
"\n",
"\n",
"\n",
"# def get_cur_time(date = ''):\n",
"# if date == '':\n",
"# now = datetime.now()\n",
"# else:\n",
"# now = date\n",
"# year = now.year\n",
"# month = now.month\n",
"# day = now.day\n",
"\n",
"# if month < 10:\n",
"# month = \"0\" + str(month)\n",
"# if day < 10:\n",
"# day = \"0\" + str(day)\n",
"# cur_time = str(year) + str(month) + str(day)\n",
"# cur_time2 = str(year) + \"-\" + str(month) + \"-\" + str(day)\n",
"# # cur_time = '20231007'\n",
"# # cur_time2 = '2023-10-07'\n",
"# return cur_time, cur_time2\n",
"\n",
"\n",
"def get_head_push_auth():\n",
@ -205,7 +289,7 @@
" \"funcOperation\": \"新增\",\n",
" \"data\": [\n",
" {\"dataItemNo\": \"C01100036|Forecast_Price|ACN\",\n",
" \"dataDate\": get_cur_time(date)[0],\n",
" \"dataDate\": getNow(date)[0],\n",
" \"dataStatus\": \"add\",\n",
" \"dataValue\": forecast_price()\n",
" }\n",
@ -217,7 +301,7 @@
" print(res.text)\n",
"\n",
"def forecast_price():\n",
" df_test = pd.read_excel('沥青数据项.xlsx',sheet_name='数据项历史数据')\n",
" df_test = pd.read_excel('沥青数据项.xlsx')\n",
" df_test.drop([0],inplace=True)\n",
" df_test[['汽油执行价','柴油执行价','齐鲁石化销量','齐鲁石化产量','齐鲁石化成交价','齐鲁石化库存','科力达销量',\n",
" '科力达产量','科力达成交价','科力达库存','弘润销量','弘润产量','弘润成交价','弘润库存','市场成交价','京博指导价',\n",
@ -277,7 +361,7 @@
"\n",
" pd.set_option('display.max_rows',40) \n",
" pd.set_option('display.max_columns',40) \n",
" df_test = pd.read_excel('沥青数据项.xlsx',sheet_name='数据项历史数据')\n",
" df_test = pd.read_excel('沥青数据项.xlsx')\n",
" df_test.drop([0],inplace=True)\n",
" df_test[['汽油执行价','柴油执行价','齐鲁石化销量','齐鲁石化产量','齐鲁石化成交价','齐鲁石化库存','科力达销量',\n",
" '科力达产量','科力达成交价','科力达库存','弘润销量','弘润产量','弘润成交价','弘润库存','市场成交价','京博指导价',\n",
@ -434,31 +518,40 @@
" pickle.dump(grid_search_XGB, file)\n",
"\n",
"def read_xls_data():\n",
" \"\"\"获取特征项ID\"\"\"\n",
" global one_cols, two_cols\n",
" # 打开 XLS 文件\n",
" workbook = xlrd.open_workbook(read_file_path_name)\n",
" # 使用pandas读取Excel文件\n",
" df = pd.read_excel(read_file_path_name, header=None) # 不自动识别列名\n",
" # 获取第二行数据（索引为1）\n",
" one_cols = df.iloc[1].tolist()[1:]\n",
" print(f'获取到的数据项ID{one_cols}')\n",
"\n",
" # 获取所有表格名称\n",
" # sheet_names = workbook.sheet_names()\n",
"# def read_xls_data():\n",
"# global one_cols, two_cols\n",
"# # 打开 XLS 文件\n",
"# workbook = xlrd.open_workbook(read_file_path_name)\n",
"\n",
" # 选择第一个表格\n",
" sheet = workbook.sheet_by_index(0)\n",
"# # 获取所有表格名称\n",
"# # sheet_names = workbook.sheet_names()\n",
"\n",
" # 获取行数和列数\n",
" num_rows = sheet.nrows\n",
" # num_cols = sheet.ncols\n",
"# # 选择第一个表格\n",
"# sheet = workbook.sheet_by_index(0)\n",
"\n",
" # 遍历每一行，获取单元格数据\n",
" # for i in range(num_rows):\n",
" # row_data = sheet.row_values(i)\n",
" # one_cols.append(row_data)\n",
" # two_cols.append(row_data[1])\n",
"# # 获取行数和列数\n",
"# num_rows = sheet.nrows\n",
"# # num_cols = sheet.ncols\n",
"\n",
" row_data = sheet.row_values(1)\n",
" one_cols = row_data\n",
"# # 遍历每一行，获取单元格数据\n",
"# # for i in range(num_rows):\n",
"# # row_data = sheet.row_values(i)\n",
"# # one_cols.append(row_data)\n",
"# # two_cols.append(row_data[1])\n",
"\n",
" # 关闭 XLS 文件\n",
" # workbook.close()\n",
"# row_data = sheet.row_values(1)\n",
"# one_cols = row_data\n",
"\n",
"# # 关闭 XLS 文件\n",
"# # workbook.close()\n",
"\n",
"def start_3(date=None,token=None,token_push=None):\n",
" '''预测上传数据'''\n",
@ -570,74 +663,111 @@
" else:\n",
" return None\n",
"\n",
"def save_queryDataListItemNos_xls(data_df,dataItemNoList):\n",
"\n",
"def save_queryDataListItemNos_xls(data_df,dataItemNoList):\n",
" current_year_month = datetime.now().strftime('%Y-%m')\n",
" grouped = data_df.groupby(\"dataDate\")\n",
"\n",
" # 打开xls文件\n",
" workbook = xlrd.open_workbook('沥青数据项.xls')\n",
"\n",
" # 获取所有sheet的个数\n",
" sheet_count = len(workbook.sheet_names())\n",
"\n",
" # 获取所有sheet的名称\n",
" sheet_names = workbook.sheet_names()\n",
"\n",
" new_workbook = xlwt.Workbook()\n",
" for i in range(sheet_count):\n",
" # 获取当前sheet\n",
" sheet = workbook.sheet_by_index(i)\n",
"\n",
" # 获取sheet的行数和列数\n",
" row_count = sheet.nrows\n",
" col_count = sheet.ncols\n",
" # 获取原有数据\n",
" data = []\n",
" for row in range(row_count):\n",
" row_data = []\n",
" for col in range(col_count):\n",
" row_data.append(sheet.cell_value(row, col))\n",
" data.append(row_data)\n",
"\n",
" # 创建xlwt的Workbook对象\n",
" # 创建sheet\n",
" new_sheet = new_workbook.add_sheet(sheet_names[i])\n",
"\n",
" # 使用openpyxl打开xlsx文件\n",
" from openpyxl import load_workbook\n",
" workbook = load_workbook('沥青数据项.xlsx')\n",
"\n",
" # 创建新工作簿\n",
" new_workbook = load_workbook('沥青数据项.xlsx')\n",
" \n",
" for sheetname in workbook.sheetnames:\n",
" sheet = workbook[sheetname]\n",
" new_sheet = new_workbook[sheetname]\n",
" \n",
" current_year_month_row = 0\n",
" # 将原有的数据写入新的sheet\n",
" for row in range(row_count):\n",
" for col in range(col_count):\n",
" col0 = data[row][0]\n",
" # print(\"col0\",col0[:7])\n",
" if col0[:7] == current_year_month:\n",
" current_year_month_row += 1\n",
" break\n",
" new_sheet.write(row, col, data[row][col])\n",
" # 查找当前月份数据起始行\n",
" for row_idx, row in enumerate(sheet.iter_rows(values_only=True), 1):\n",
" if str(row[0]).startswith(current_year_month):\n",
" current_year_month_row += 1\n",
"\n",
"\n",
" # print(\"current_year_month_row\",current_year_month_row)\n",
" if i == 0:\n",
" rowFlag = 0\n",
" # 查看每组数据\n",
" for date, group in grouped:\n",
" new_sheet.write(row_count + rowFlag - current_year_month_row, 0, date)\n",
" for j in range(len(dataItemNoList)):\n",
" dataItemNo = dataItemNoList[j]\n",
"\n",
" # for dataItemNo in dataItemNoList:\n",
" # 追加新数据\n",
" if sheetname == workbook.sheetnames[0]:\n",
" start_row = sheet.max_row - current_year_month_row + 1\n",
" for row_idx, (date, group) in enumerate(grouped, start=start_row):\n",
" new_sheet.cell(row=row_idx, column=1, value=date)\n",
" for j, dataItemNo in enumerate(dataItemNoList, start=2):\n",
" if group[group[\"dataItemNo\"] == dataItemNo][\"dataValue\"].values:\n",
" new_sheet.cell(row=row_idx, column=j, \n",
" value=group[group[\"dataItemNo\"] == dataItemNo][\"dataValue\"].values[0])\n",
"\n",
" new_sheet.write(row_count + rowFlag - current_year_month_row, j + 1, group[group[\"dataItemNo\"] == dataItemNo][\"dataValue\"].values[0])\n",
"\n",
" rowFlag += 1\n",
"\n",
"\n",
" # 保存新的xls文件\n",
" # 保存修改后的xlsx文件\n",
" new_workbook.save(\"沥青数据项.xlsx\")\n",
"\n",
"\n",
"# def save_queryDataListItemNos_xls(data_df,dataItemNoList):\n",
"# from datetime import datetime, timedelta\n",
"# current_year_month = datetime.now().strftime('%Y-%m')\n",
"# grouped = data_df.groupby(\"dataDate\")\n",
"\n",
"# # 打开xls文件\n",
"# workbook = xlrd.open_workbook('沥青数据项.xlsx')\n",
"\n",
"# # 获取所有sheet的个数\n",
"# sheet_count = len(workbook.sheet_names())\n",
"\n",
"# # 获取所有sheet的名称\n",
"# sheet_names = workbook.sheet_names()\n",
"\n",
"# new_workbook = xlwt.Workbook()\n",
"# for i in range(sheet_count):\n",
"# # 获取当前sheet\n",
"# sheet = workbook.sheet_by_index(i)\n",
"\n",
"# # 获取sheet的行数和列数\n",
"# row_count = sheet.nrows\n",
"# col_count = sheet.ncols\n",
"# # 获取原有数据\n",
"# data = []\n",
"# for row in range(row_count):\n",
"# row_data = []\n",
"# for col in range(col_count):\n",
"# row_data.append(sheet.cell_value(row, col))\n",
"# data.append(row_data)\n",
"\n",
"# # 创建xlwt的Workbook对象\n",
"# # 创建sheet\n",
"# new_sheet = new_workbook.add_sheet(sheet_names[i])\n",
"\n",
"\n",
"# current_year_month_row = 0\n",
"# # 将原有的数据写入新的sheet\n",
"# for row in range(row_count):\n",
"# for col in range(col_count):\n",
"# col0 = data[row][0]\n",
"# # print(\"col0\",col0[:7])\n",
"# if col0[:7] == current_year_month:\n",
"# current_year_month_row += 1\n",
"# break\n",
"# new_sheet.write(row, col, data[row][col])\n",
"\n",
"\n",
"# # print(\"current_year_month_row\",current_year_month_row)\n",
"# if i == 0:\n",
"# rowFlag = 0\n",
"# # 查看每组数据\n",
"# for date, group in grouped:\n",
"# new_sheet.write(row_count + rowFlag - current_year_month_row, 0, date)\n",
"# for j in range(len(dataItemNoList)):\n",
"# dataItemNo = dataItemNoList[j]\n",
"\n",
"# # for dataItemNo in dataItemNoList:\n",
"# if group[group[\"dataItemNo\"] == dataItemNo][\"dataValue\"].values:\n",
"\n",
"# new_sheet.write(row_count + rowFlag - current_year_month_row, j + 1, group[group[\"dataItemNo\"] == dataItemNo][\"dataValue\"].values[0])\n",
"\n",
"# rowFlag += 1\n",
"\n",
"\n",
"# # 保存新的xls文件\n",
"# new_workbook.save(\"沥青数据项.xlsx\")\n",
"\n",
"def queryDataListItemNos(token=None):\n",
" from datetime import datetime, timedelta\n",
" df = pd.read_excel('沥青数据项.xlsx')\n",
" dataItemNoList = df.iloc[0].tolist()[1:]\n",
" \n",
@ -668,7 +798,130 @@
" save_queryDataListItemNos_xls(data_df,dataItemNoList)\n",
" print('当月数据更新完成')\n",
"\n",
"\n",
"def save_xls_1(append_rows):\n",
"\n",
" # 打开xls文件\n",
" workbook = xlrd.open_workbook('沥青数据项.xlsx')\n",
"\n",
" # 获取所有sheet的个数\n",
" sheet_count = len(workbook.sheet_names())\n",
"\n",
" # 获取所有sheet的名称\n",
" sheet_names = workbook.sheet_names()\n",
"\n",
" new_workbook = xlwt.Workbook()\n",
" for i in range(sheet_count):\n",
" # 获取当前sheet\n",
" sheet = workbook.sheet_by_index(i)\n",
"\n",
" # 获取sheet的行数和列数\n",
" row_count = sheet.nrows - 1\n",
" col_count = sheet.ncols\n",
" # 获取原有数据\n",
" data = []\n",
" for row in range(row_count):\n",
" row_data = []\n",
" for col in range(col_count):\n",
" row_data.append(sheet.cell_value(row, col))\n",
" data.append(row_data)\n",
" # 创建xlwt的Workbook对象\n",
" # 创建sheet\n",
" new_sheet = new_workbook.add_sheet(sheet_names[i])\n",
"\n",
" # 将原有的数据写入新的sheet\n",
" for row in range(row_count):\n",
" for col in range(col_count):\n",
" new_sheet.write(row, col, data[row][col])\n",
"\n",
" if i == 0:\n",
" # 在新的sheet中添加数据\n",
" for col in range(col_count):\n",
" new_sheet.write(row_count, col, append_rows[col])\n",
"\n",
" # 保存新的xls文件\n",
" new_workbook.save(\"沥青数据项.xlsx\") \n",
"\n",
"\n",
"\n",
"def start(date=''):\n",
" \"\"\"获取当日数据\"\"\"\n",
" read_xls_data()\n",
" token = get_head_auth()\n",
" if not token:\n",
" return\n",
" \n",
" cur_time,cur_time2 = getNow(date)\n",
" print(f\"获取{cur_time}数据\")\n",
" datas = get_data_value(token, one_cols,date=cur_time)\n",
" print(len(datas))\n",
" print(datas)\n",
" if not datas:\n",
" return\n",
"\n",
" append_rows = [cur_time2]\n",
" dataItemNo_dataValue = {}\n",
" for data_value in datas:\n",
" if \"dataValue\" not in data_value:\n",
" print(data_value)\n",
" dataItemNo_dataValue[data_value[\"dataItemNo\"]] = \"\"\n",
" else:\n",
" dataItemNo_dataValue[data_value[\"dataItemNo\"]] = data_value[\"dataValue\"]\n",
" \n",
" for value in one_cols:\n",
" if value in dataItemNo_dataValue:\n",
" append_rows.append(dataItemNo_dataValue[value])\n",
" else:\n",
" append_rows.append(\"\")\n",
" \n",
" print('添加的行：',len(append_rows),append_rows)\n",
" save_xls_2(append_rows)\n",
"\n",
"\n",
"def save_xls_2(append_rows):\n",
" \"\"\"保存或更新数据到Excel文件\n",
" 参数:\n",
" append_rows (list): 需要追加/更新的数据行，格式为[日期, 数据项1, 数据项2,...]\n",
" \"\"\"\n",
" # try:\n",
" # 读取现有数据（假设第一行为列名）\n",
" df = pd.read_excel('沥青数据项.xlsx', sheet_name=0)\n",
" print('文件中的数据列数：',len(df.columns),df.columns)\n",
" # 转换append_rows为DataFrame\n",
" if len(append_rows) != len(df.columns):\n",
" # 去除第二个元素 ，不知道什么原因多一个空数据\n",
" append_rows.pop(1)\n",
" append_rows = pd.DataFrame([append_rows],columns=df.columns)\n",
" # 创建新数据行\n",
" new_date = append_rows['日期'].values[0]\n",
" \n",
" dates = df['日期'].to_list()\n",
" # 判断日期是否存在\n",
" if new_date in dates:\n",
" # 找到日期所在行的索引\n",
" date_mask = df['日期'] == new_date\n",
" # 存在则更新数据\n",
" df.loc[date_mask] = append_rows.values\n",
" print(f\"更新 {new_date} 数据\")\n",
" else:\n",
" # 不存在则追加数据\n",
" df = pd.concat([df, append_rows], ignore_index=True)\n",
" print(df.head())\n",
" print(df.tail())\n",
" print(f\"插入 {new_date} 新数据\")\n",
" \n",
" # 保存更新后的数据\n",
" df.to_excel('沥青数据项.xlsx', index=False, engine='openpyxl')\n",
" \n",
" # except FileNotFoundError:\n",
" # # 如果文件不存在则创建新文件\n",
" # pd.DataFrame([append_rows]).to_excel('沥青数据项.xlsx', index=False, engine='openpyxl')\n",
" # except Exception as e:\n",
" # print(f\"保存数据时发生错误: {str(e)}\")\n",
"\n",
"\n",
"def main(start_date=None,token=None,token_push=None):\n",
" from datatime import datetime\n",
" if start_date is None:\n",
" start_date = datetime.now()\n",
" if token is None:\n",
@ -677,63 +930,192 @@
" token_push = get_head_push_auth()\n",
" date = start_date.strftime('%Y%m%d')\n",
" print(date)\n",
"# start(date)\n",
" # 更新当月数据\n",
" queryDataListItemNos(token)\n",
" # 训练模型\n",
" optimize_Model()\n",
" # 预测&上传预测结果\n",
" # # 预测&上传预测结果\n",
" upload_data_to_system(token_push,start_date)"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"metadata": {},
"outputs": [
{
<<<<<<< .mine
"ename": "AttributeError",
"evalue": "module 'datetime' has no attribute 'now'",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[1;32mIn[5], line 20\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;66;03m# if __name__ == \"__main__\":\u001b[39;00m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;66;03m# print(\"运行中ing...\")\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;66;03m# # 每天定时12点运行\u001b[39;00m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[38;5;66;03m# except :\u001b[39;00m\n\u001b[0;32m 19\u001b[0m \u001b[38;5;66;03m# print(f\"{current_time}任务失败\")\u001b[39;00m\n\u001b[1;32m---> 20\u001b[0m main()\n",
"Cell \u001b[1;32mIn[1], line 848\u001b[0m, in \u001b[0;36mmain\u001b[1;34m(start_date, token, token_push)\u001b[0m\n\u001b[0;32m 846\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmain\u001b[39m(start_date\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,token\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,token_push\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m 847\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m start_date \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m--> 848\u001b[0m start_date \u001b[38;5;241m=\u001b[39m datetime\u001b[38;5;241m.\u001b[39mnow()\n\u001b[0;32m 849\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m token \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m 850\u001b[0m token \u001b[38;5;241m=\u001b[39m get_head_auth()\n",
"\u001b[1;31mAttributeError\u001b[0m: module 'datetime' has no attribute 'now'"
||||||| .r87044
"name": "stdout",
"output_type": "stream",
"text": [
"运行中ing...\n"
=======
"name": "stdout",
"output_type": "stream",
"text": [
"20250522\n"
>>>>>>> .r87201
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_20588\\2014406737.py:615: DeprecationWarning:\n",
"\n",
"The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"当月数据更新完成\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_20588\\2014406737.py:298: UserWarning:\n",
"\n",
"The argument 'infer_datetime_format' is deprecated and will be removed in a future version. A strict version of it is now the default, see https://pandas.pydata.org/pdeps/0004-consistent-to-datetime-parsing.html. You can safely remove this argument.\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using matplotlib backend: inline\n",
"%pylab is deprecated, use %matplotlib inline and import the required libraries.\n",
"Populating the interactive namespace from numpy and matplotlib\n",
"Fitting 3 folds for each of 180 candidates, totalling 540 fits\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\EDY\\.conda\\envs\\jiageyuce\\Lib\\site-packages\\IPython\\core\\magics\\pylab.py:166: UserWarning:\n",
"\n",
"pylab import has clobbered these variables: ['random', 'datetime', 'plot', '__version__']\n",
"`%matplotlib` prevents importing * from pylab and numpy\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Best score: 0.997\n",
"Best parameters set:\n",
"\tlearning_rate: 0.1\n",
"\tmax_depth: 8\n",
"\tn_estimators: 90\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_20588\\2014406737.py:238: UserWarning:\n",
"\n",
"The argument 'infer_datetime_format' is deprecated and will be removed in a future version. A strict version of it is now the default, see https://pandas.pydata.org/pdeps/0004-consistent-to-datetime-parsing.html. You can safely remove this argument.\n",
"\n",
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_20588\\2014406737.py:272: FutureWarning:\n",
"\n",
"Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"日期\n",
"2025-05-22 3600.097412\n",
"Name: 日度预测价格, dtype: float32\n",
"{\"confirmFlg\":false,\"status\":true}\n"
]
}
],
"source": [
"\n",
"if __name__ == \"__main__\":\n",
" print(\"运行中ing...\")\n",
" # 每天定时12点运行\n",
" while True:\n",
" # 获取当前时间\n",
" current_time = time.strftime(\"%H:%M:%S\", time.localtime())\n",
" # print(current_time)\n",
"# if __name__ == \"__main__\":\n",
"# print(\"运行中ing...\")\n",
"# # 每天定时12点运行\n",
"# while True:\n",
"# # 获取当前时间\n",
"# current_time = time.strftime(\"%H:%M:%S\", time.localtime())\n",
"# # print(current_time)\n",
"\n",
<<<<<<< .mine
"# # 判断当前时间是否为执行任务的时间点\n",
"# try:\n",
"# if current_time == \"09:13:00\":\n",
"# print(\"执行定时任务\")\n",
"# main()\n",
"# elif current_time == \"09:18:00\":\n",
"# print(\"执行定时任务\")\n",
"# main()\n",
"# time.sleep(1)\n",
"# except :\n",
"# print(f\"{current_time}任务失败\")\n",
"main()\n",
"main()\n",
||||||| .r87044
" # 判断当前时间是否为执行任务的时间点\n",
" try:\n",
" if current_time == \"09:15:00\":\n",
" if current_time == \"09:13:00\":\n",
" print(\"执行定时任务\")\n",
" main()\n",
" elif current_time == \"09:18:00\":\n",
" print(\"执行定时任务\")\n",
" main()\n",
" elif current_time == \"20:00:00\":\n",
" print('更新数据')\n",
" start_3()\n",
" time.sleep(1)\n",
" except :\n",
" print(f\"{current_time}任务失败\")\n",
"\n",
"\n",
"# main()\n",
=======
"# # 判断当前时间是否为执行任务的时间点\n",
"# try:\n",
"# if current_time == \"09:13:00\":\n",
"# print(\"执行定时任务\")\n",
"# main()\n",
"# elif current_time == \"09:18:00\":\n",
"# print(\"执行定时任务\")\n",
"# main()\n",
"# time.sleep(1)\n",
"# except :\n",
"# print(f\"{current_time}任务失败\")\n",
"main()\n",
>>>>>>> .r87201
" # 检测数据准确性, 需要检测放开\n",
" # check_data(\"100028098|LISTING_PRICE\")\n",
" # check_data(\"9137070016544622XB|DAY_Yield\")\n"
" # check_data(\"9137070016544622XB|DAY_Yield\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"# start_date = datetime(2025, 4, 1)\n",
"# end_date = datetime(2025, 5, 7)\n",
"# start_date = datetime(2025, 5, 16)\n",
"# end_date = datetime(2025, 5, 19)\n",
"# token = get_head_auth()\n",
"\n",
"# token_push = get_head_push_auth()\n",
@ -745,7 +1127,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
@ -753,86 +1135,9 @@
"\n",
"\n",
"class codeBackup:\n",
" def save_xls_1(append_rows):\n",
"\n",
" # 打开xls文件\n",
" workbook = xlrd.open_workbook('沥青数据项.xlsx')\n",
"\n",
" # 获取所有sheet的个数\n",
" sheet_count = len(workbook.sheet_names())\n",
"\n",
" # 获取所有sheet的名称\n",
" sheet_names = workbook.sheet_names()\n",
"\n",
" new_workbook = xlwt.Workbook()\n",
" for i in range(sheet_count):\n",
" # 获取当前sheet\n",
" sheet = workbook.sheet_by_index(i)\n",
"\n",
" # 获取sheet的行数和列数\n",
" row_count = sheet.nrows - 1\n",
" col_count = sheet.ncols\n",
" # 获取原有数据\n",
" data = []\n",
" for row in range(row_count):\n",
" row_data = []\n",
" for col in range(col_count):\n",
" row_data.append(sheet.cell_value(row, col))\n",
" data.append(row_data)\n",
" # 创建xlwt的Workbook对象\n",
" # 创建sheet\n",
" new_sheet = new_workbook.add_sheet(sheet_names[i])\n",
"\n",
" # 将原有的数据写入新的sheet\n",
" for row in range(row_count):\n",
" for col in range(col_count):\n",
" new_sheet.write(row, col, data[row][col])\n",
"\n",
" if i == 0:\n",
" # 在新的sheet中添加数据\n",
" for col in range(col_count):\n",
" new_sheet.write(row_count, col, append_rows[col])\n",
"\n",
" # 保存新的xls文件\n",
" new_workbook.save(\"沥青数据项.xlsx\") \n",
"\n",
" \n",
"\n",
" def start_2(date,token):\n",
" '''更新数据'''\n",
" read_xls_data()\n",
"\n",
" # token = get_head_auth()\n",
" # if not token:\n",
" # return\n",
" \n",
"\n",
" datas = get_data_value(token, one_cols[1:],date)\n",
" if not datas:\n",
" return\n",
"\n",
" # data_list = [two_cols, one_cols]\n",
" append_rows = [get_cur_time(date=date)[1]]\n",
" dataItemNo_dataValue = {}\n",
" for data_value in datas:\n",
" if \"dataValue\" not in data_value:\n",
" print(data_value)\n",
" dataItemNo_dataValue[data_value[\"dataItemNo\"]] = \"\"\n",
" else:\n",
" dataItemNo_dataValue[data_value[\"dataItemNo\"]] = data_value[\"dataValue\"]\n",
" \n",
" for value in one_cols[1:]:\n",
" if value in dataItemNo_dataValue:\n",
" append_rows.append(dataItemNo_dataValue[value])\n",
" else:\n",
" append_rows.append(\"\")\n",
" print('新增数据：',append_rows)\n",
" save_xls_1(append_rows)\n",
"\n",
" \n",
" # data_list.append(three_cols)\n",
" # write_xls(data_list)\n",
" \n",
"\n",
" def write_xls(data,date):\n",
" # 创建一个Workbook对象\n",
@ -932,7 +1237,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "base",
"language": "python",
"name": "python3"
},
@ -946,7 +1251,13 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
<<<<<<< .mine
"version": "3.11.7"
||||||| .r87044
"version": "3.7.0"
=======
"version": "3.11.11"
>>>>>>> .r87201
}
},
"nbformat": 4,

1100
aisenzhecode/沥青/沥青定量价格预测每日推送-ytj.ipynb.mine Normal file
View File

File diff suppressed because it is too large Load Diff

1095
aisenzhecode/沥青/沥青定量价格预测每日推送-ytj.ipynb.r87044 Normal file
View File

File diff suppressed because it is too large Load Diff

1156
aisenzhecode/沥青/沥青定量价格预测每日推送-ytj.ipynb.r87201 Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
aisenzhecode/沥青/沥青数据项.xlsx
View File

Binary file not shown.

BIN
aisenzhecode/沥青/沥青数据项.xlsx.r87044 Normal file
View File

Binary file not shown.

BIN
aisenzhecode/沥青/沥青数据项.xlsx.r87201 Normal file
View File

Binary file not shown.

BIN
aisenzhecode/液化石油气/日度价格预测_液化气最佳模型.pkl
View File

Binary file not shown.

178
aisenzhecode/液化石油气/液化气价格预测ytj.ipynb
View File

@ -2,28 +2,20 @@
"cells": [
{
"cell_type": "code",
"execution_count": null,
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From C:\\Users\\EDY\\AppData\\Roaming\\Python\\Python311\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
"\n"
]
},
{
"data": {
"text/html": [
" <script type=\"text/javascript\">\n",
" window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
" if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (typeof require !== 'undefined') {\n",
" require.undef(\"plotly\");\n",
" requirejs.config({\n",
" paths: {\n",
" 'plotly': ['https://cdn.plot.ly/plotly-2.12.1.min']\n",
" 'plotly': ['https://cdn.plot.ly/plotly-2.2.0.min']\n",
" }\n",
" });\n",
" require(['plotly'], function(Plotly) {\n",
@ -687,10 +679,10 @@
" # 更新当月数据\n",
" queryDataListItemNos(start_date,token)\n",
" # 更新当日数据\n",
" start(date)\n",
"# start(date)\n",
" # 训练模型\n",
" optimize_Model()\n",
" # 预测&上传预测结果\n",
" # # 预测&上传预测结果\n",
" upload_data_to_system(token_push,start_date)\n",
"\n"
]
@ -707,8 +699,18 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"运行中ing...\n"
]
}
],
"source": [
"\n",
"if __name__ == \"__main__\":\n",
@ -717,18 +719,18 @@
" while True:\n",
" # 获取当前时间\n",
" current_time = time.strftime(\"%H:%M:%S\", time.localtime())\n",
" # try:\n",
" try:\n",
" # 判断当前时间是否为执行任务的时间点\n",
" if current_time == \"09:15:00\":\n",
" # if current_time == \"17:12:00\":\n",
" print(\"执行定时任务\")\n",
" main()\n",
" elif current_time == \"20:00:00\":\n",
" start_1()\n",
" time.sleep(1)\n",
" # except:\n",
" # print(f\"{current_time}执行失败\")\n",
"\n",
" if current_time == \"09:15:00\":\n",
" print(\"执行定时任务\")\n",
" main()\n",
" elif current_time == \"09:20:00\":\n",
" print(\"执行定时任务\")\n",
" main()\n",
" time.sleep(1)\n",
" except:\n",
" print(f\"{current_time}执行失败\")\n",
"# main() \n",
" # 检测数据准确性, 需要检测放开\n",
" # check_data(\"100028098|LISTING_PRICE\")\n",
" # check_data(\"9137070016544622XB|DAY_Yield\")\n"
@ -738,134 +740,22 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"获取的token: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhcGlfZGV2IiwidGgiOiI4YTQ1NzdkYmQ5MTk2NzU3NThkNTc5OTlhMWU4OTFmZSIsImx0IjoiYXBpIiwiaXNzIjoiIiwidG0iOiJQQyIsImV4cCI6MTc0NzMwNzgyMCwianRpIjoiMzkyNmRlYjI3YWJkNDIyNjk2MDFmMGE4MTFjZWM4YzgifQ.N0sWMfEvZe0Jh7xPyBlLWnYqYLOLQdiRB3JASMSd0vw\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_25852\\3372978512.py:602: DeprecationWarning:\n",
"\n",
"The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"当月数据更新完成\n",
"20250514\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_25852\\3372978512.py:602: DeprecationWarning:\n",
"\n",
"The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"当月数据更新完成\n",
"获取到的数据项ID['YHQMXBB|C01100008|STRIKE_PRICE', 'C01100008|CORTED_VALUE', 'C01100008|AUCTION_MAX_PRICE', 'C01100008|AMOUNT', 'C01100008|YEDAY_AMOUNT', '100028046|LISTING_PRICE', 'ICE_CL0_LAST_YEDAY_PRICE', '91370200163576944B|C01100008|STRIKE_PRICE', '9137078672073757X8|C01100008|STRIKE_PRICE', '91370500674526498A|C01100008|STRIKE_PRICE', '91370305773165341A|C01100008|STRIKE_PRICE', '91370521164880008P|C01100008|STRIKE_PRICE', '91370321164425136B|C01100008|STRIKE_PRICE', 'SD|GC|ZDW|LIST_PRICE', '370500|ISOBUTANE|LIST_PRICE', 'SD|YT|SG|LIST_PRICE', '91110000710926094P|C01100008|SUPPLY_MERE', '91110000710932515R|C01100008|SUPPLY_MERE', '91370500674526498A|C01100008|SUPPLY_MERE', '91370321164425136B|C01100008|SUPPLY_MERE', 'C01100008|OTHER|SUPPLY_MERE', 'SD|WJH|DEMANDS', 'C01100008|SUY_DED_DAP', 'C01100008|EFFECTIVE_STOCK', '912102117169477344|C01100008|STRIKE_PRICE', '91110304102767480H|C01100008|STRIKE_PRICE', '91130193670310403L|C01100008|STRIKE_PRICE', 'HD|LPG|IMPORT_PRICE', 'SD|WJH|SALES_PRICE']\n",
"获取的token: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhcGlfZGV2IiwidGgiOiI4YTQ1NzdkYmQ5MTk2NzU3NThkNTc5OTlhMWU4OTFmZSIsImx0IjoiYXBpIiwiaXNzIjoiIiwidG0iOiJQQyIsImV4cCI6MTc0NzMwNzgyMywianRpIjoiZGZhZjM2ZjcyNWRhNDJmMThjYThmZWE0Y2VkNjQ1MjcifQ.IRHbVrERfphVjgcVRZvnpUN6GBMIH4uCQ9SIcS5U3fA\n",
"获取20250514数据\n",
"数据项查询参数search_data：\n",
"{'data': {'date': '20250514', 'dataItemNoList': ['YHQMXBB|C01100008|STRIKE_PRICE', 'C01100008|CORTED_VALUE', 'C01100008|AUCTION_MAX_PRICE', 'C01100008|AMOUNT', 'C01100008|YEDAY_AMOUNT', '100028046|LISTING_PRICE', 'ICE_CL0_LAST_YEDAY_PRICE', '91370200163576944B|C01100008|STRIKE_PRICE', '9137078672073757X8|C01100008|STRIKE_PRICE', '91370500674526498A|C01100008|STRIKE_PRICE', '91370305773165341A|C01100008|STRIKE_PRICE', '91370521164880008P|C01100008|STRIKE_PRICE', '91370321164425136B|C01100008|STRIKE_PRICE', 'SD|GC|ZDW|LIST_PRICE', '370500|ISOBUTANE|LIST_PRICE', 'SD|YT|SG|LIST_PRICE', '91110000710926094P|C01100008|SUPPLY_MERE', '91110000710932515R|C01100008|SUPPLY_MERE', '91370500674526498A|C01100008|SUPPLY_MERE', '91370321164425136B|C01100008|SUPPLY_MERE', 'C01100008|OTHER|SUPPLY_MERE', 'SD|WJH|DEMANDS', 'C01100008|SUY_DED_DAP', 'C01100008|EFFECTIVE_STOCK', '912102117169477344|C01100008|STRIKE_PRICE', '91110304102767480H|C01100008|STRIKE_PRICE', '91130193670310403L|C01100008|STRIKE_PRICE', 'HD|LPG|IMPORT_PRICE', 'SD|WJH|SALES_PRICE']}, 'funcModule': '数据项', 'funcOperation': '查询'}\n",
"数据项查询结果search_res：\n",
"{\"confirmFlg\":false,\"data\":[{\"dataDate\":\"20250514\",\"dataItemNo\":\"100028046|LISTING_PRICE\",\"dataValue\":7613.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"91110000710926094P|C01100008|SUPPLY_MERE\",\"dataValue\":1300.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"91110000710932515R|C01100008|SUPPLY_MERE\"},{\"dataDate\":\"20250514\",\"dataItemNo\":\"91370321164425136B|C01100008|STRIKE_PRICE\",\"dataValue\":4650.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"91370321164425136B|C01100008|SUPPLY_MERE\",\"dataValue\":300.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"91370500674526498A|C01100008|SUPPLY_MERE\",\"dataValue\":175.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"91370521164880008P|C01100008|STRIKE_PRICE\",\"dataValue\":4705.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"C01100008|AUCTION_MAX_PRICE\",\"dataValue\":4650.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"C01100008|CORTED_VALUE\",\"dataValue\":4650.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"C01100008|OTHER|SUPPLY_MERE\",\"dataValue\":5000.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"C01100008|SUY_DED_DAP\",\"dataValue\":-50.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"ICE_CL0_LAST_YEDAY_PRICE\",\"dataValue\":66.60000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"SD|WJH|DEMANDS\",\"dataValue\":8500.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"SD|WJH|SALES_PRICE\",\"dataValue\":7900.00000000},{\"dataDate\":\"20250514\",\"dataItemNo\":\"YHQMXBB|C01100008|STRIKE_PRICE\",\"dataValue\":4650.00000000}],\"status\":true}\n",
"数据项查询结果: [{'dataDate': '20250514', 'dataItemNo': '100028046|LISTING_PRICE', 'dataValue': 7613.0}, {'dataDate': '20250514', 'dataItemNo': '91110000710926094P|C01100008|SUPPLY_MERE', 'dataValue': 1300.0}, {'dataDate': '20250514', 'dataItemNo': '91110000710932515R|C01100008|SUPPLY_MERE'}, {'dataDate': '20250514', 'dataItemNo': '91370321164425136B|C01100008|STRIKE_PRICE', 'dataValue': 4650.0}, {'dataDate': '20250514', 'dataItemNo': '91370321164425136B|C01100008|SUPPLY_MERE', 'dataValue': 300.0}, {'dataDate': '20250514', 'dataItemNo': '91370500674526498A|C01100008|SUPPLY_MERE', 'dataValue': 175.0}, {'dataDate': '20250514', 'dataItemNo': '91370521164880008P|C01100008|STRIKE_PRICE', 'dataValue': 4705.0}, {'dataDate': '20250514', 'dataItemNo': 'C01100008|AUCTION_MAX_PRICE', 'dataValue': 4650.0}, {'dataDate': '20250514', 'dataItemNo': 'C01100008|CORTED_VALUE', 'dataValue': 4650.0}, {'dataDate': '20250514', 'dataItemNo': 'C01100008|OTHER|SUPPLY_MERE', 'dataValue': 5000.0}, {'dataDate': '20250514', 'dataItemNo': 'C01100008|SUY_DED_DAP', 'dataValue': -50.0}, {'dataDate': '20250514', 'dataItemNo': 'ICE_CL0_LAST_YEDAY_PRICE', 'dataValue': 66.6}, {'dataDate': '20250514', 'dataItemNo': 'SD|WJH|DEMANDS', 'dataValue': 8500.0}, {'dataDate': '20250514', 'dataItemNo': 'SD|WJH|SALES_PRICE', 'dataValue': 7900.0}, {'dataDate': '20250514', 'dataItemNo': 'YHQMXBB|C01100008|STRIKE_PRICE', 'dataValue': 4650.0}]\n",
"{'dataDate': '20250514', 'dataItemNo': '91110000710932515R|C01100008|SUPPLY_MERE'}\n",
"添加的行： ['2025-05-14', 4650.0, 4650.0, 4650.0, '', '', 7613.0, 66.6, '', '', '', '', 4705.0, 4650.0, '', '', '', 1300.0, '', 175.0, 300.0, 5000.0, 8500.0, -50.0, '', '', '', '', '', 7900.0]\n",
"更新 2025-05-14 数据\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_25852\\3372978512.py:238: UserWarning:\n",
"\n",
"The argument 'infer_datetime_format' is deprecated and will be removed in a future version. A strict version of it is now the default, see https://pandas.pydata.org/pdeps/0004-consistent-to-datetime-parsing.html. You can safely remove this argument.\n",
"\n",
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_25852\\3372978512.py:240: UserWarning:\n",
"\n",
"The argument 'infer_datetime_format' is deprecated and will be removed in a future version. A strict version of it is now the default, see https://pandas.pydata.org/pdeps/0004-consistent-to-datetime-parsing.html. You can safely remove this argument.\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using matplotlib backend: <object object at 0x0000024A7FD2F310>\n",
"%pylab is deprecated, use %matplotlib inline and import the required libraries.\n",
"Populating the interactive namespace from numpy and matplotlib\n",
"Fitting 3 folds for each of 180 candidates, totalling 540 fits\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"d:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\magics\\pylab.py:162: UserWarning:\n",
"\n",
"pylab import has clobbered these variables: ['random', 'plot', '__version__', 'datetime']\n",
"`%matplotlib` prevents importing * from pylab and numpy\n",
"\n",
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_25852\\3372978512.py:186: UserWarning:\n",
"\n",
"The argument 'infer_datetime_format' is deprecated and will be removed in a future version. A strict version of it is now the default, see https://pandas.pydata.org/pdeps/0004-consistent-to-datetime-parsing.html. You can safely remove this argument.\n",
"\n",
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_25852\\3372978512.py:188: UserWarning:\n",
"\n",
"The argument 'infer_datetime_format' is deprecated and will be removed in a future version. A strict version of it is now the default, see https://pandas.pydata.org/pdeps/0004-consistent-to-datetime-parsing.html. You can safely remove this argument.\n",
"\n",
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_25852\\3372978512.py:220: FutureWarning:\n",
"\n",
"Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Date\n",
"2025-05-14 4646.130371\n",
"Name: 日度预测价格, dtype: float32\n",
"预测值： 4646.13\n"
]
}
],
"outputs": [],
"source": [
"# start_date = datetime(2025, 5, 14)\n",
"# end_date = datetime(2025, 5, 15)\n",
"# start_date = datetime(2025, 5, 16)\n",
"# end_date = datetime(2025, 5, 19)\n",
"# token = get_head_auth()\n",
"\n",
"# while start_date < end_date:\n",
"# date = start_date.strftime('%Y%m%d')\n",
"# date2 = start_date.strftime('%Y-%m-%d')\n",
"# queryDataListItemNos(date=start_date,token=token)\n",
"# main(start_date=start_date,token=token,token_push=token)\n",
"# # # time.sleep(1)\n",
"# # start_1(start_date)\n",
"# start_date += timedelta(days=1)\n",
"# time.sleep(2)\n",
"\n",
"# # print(price_list)"
"# print(price_list)"
]
},
{
@ -878,7 +768,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "base",
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
@ -892,7 +782,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.7.0"
}
},
"nbformat": 4,

BIN
aisenzhecode/液化石油气/液化气数据.xlsx
View File

Binary file not shown.

BIN
aisenzhecode/石油苯/日度价格预测_最佳模型.pkl
View File

Binary file not shown.

198
aisenzhecode/石油苯/纯苯价格预测-自定义日期ytj.ipynb
View File

@ -2,28 +2,22 @@
"cells": [
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From C:\\Users\\EDY\\AppData\\Roaming\\Python\\Python311\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
"\n"
]
},
{
"data": {
"text/html": [
" <script type=\"text/javascript\">\n",
" window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
" if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
" if (typeof require !== 'undefined') {\n",
" require.undef(\"plotly\");\n",
" requirejs.config({\n",
" paths: {\n",
" 'plotly': ['https://cdn.plot.ly/plotly-2.12.1.min']\n",
" 'plotly': ['https://cdn.plot.ly/plotly-2.2.0.min']\n",
" }\n",
" });\n",
" require(['plotly'], function(Plotly) {\n",
@ -36,6 +30,13 @@
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"运行中ing\n"
]
}
],
"source": [
@ -274,7 +275,7 @@
" \"funcOperation\": \"新增\",\n",
" \"data\": [\n",
" {\"dataItemNo\": \"C01100047|FORECAST_PRICE\",\n",
" \"dataDate\": date,\n",
" \"dataDate\": getNow(date)[0],\n",
" \"dataStatus\": \"add\",\n",
" \"dataValue\": datavalue\n",
" }\n",
@ -282,9 +283,9 @@
" ]\n",
" }\n",
" print(data)\n",
" # headers = {\"Authorization\": token_push}\n",
" # res = requests.post(url=upload_url, headers=headers, json=data, timeout=(3, 5))\n",
" # print(res.text)\n",
" headers = {\"Authorization\": token_push}\n",
" res = requests.post(url=upload_url, headers=headers, json=data, timeout=(3, 5))\n",
" print(res.text)\n",
"\n",
" \n",
" \n",
@ -736,7 +737,7 @@
"\n",
"\n",
"def save_queryDataListItemNos_xls(data_df,dataItemNoList):\n",
"\n",
" from datetime import datetime,timedelta\n",
" current_year_month = datetime.now().strftime('%Y-%m')\n",
" grouped = data_df.groupby(\"dataDate\")\n",
"\n",
@ -803,56 +804,56 @@
"\n",
"\n",
"\n",
"\n",
"def queryDataListItemNos(token=None):\n",
"def queryDataListItemNos(date=None,token=None):\n",
" from datetime import datetime, timedelta\n",
" df = pd.read_excel('纯苯数据项.xls')\n",
" dataItemNoList = df.iloc[0].tolist()[1:]\n",
"\n",
" if token == None:\n",
" if token is None:\n",
" token = get_head_auth()\n",
" if not token:\n",
" print('token获取失败')\n",
" return\n",
" # 获取当前日期\n",
" current_date = datetime.now()\n",
"\n",
" if date is None:\n",
" current_date = datetime.now()\n",
" else:\n",
" current_date = date\n",
" # 获取当月1日\n",
" first_day_of_month = current_date.replace(day=1)\n",
"\n",
" # 格式化为 YYYYMMDD 格式\n",
" dateEnd = current_date.strftime('%Y%m%d')\n",
" dateStart = first_day_of_month.strftime('%Y%m%d')\n",
"\n",
" search_value = get_queryDataListItemNos_value(token, queryDataListItemNos_url, dataItemNoList, dateStart, dateEnd)\n",
"\n",
"\n",
" data_df = pd.DataFrame(search_value)\n",
"\n",
" data_df[\"dataDate\"] = pd.to_datetime(data_df[\"dataDate\"])\n",
" data_df[\"dataDate\"] = data_df[\"dataDate\"].dt.strftime('%Y-%m-%d')\n",
" save_queryDataListItemNos_xls(data_df,dataItemNoList)\n",
" print('当月数据更新完成')\n",
"\n",
"\n",
"# if __name__ == \"__main__\":\n",
"# print('运行中ing')\n",
"if __name__ == \"__main__\":\n",
" print('运行中ing')\n",
"\n",
"# # 每天定时12点运行\n",
"# while True:\n",
"# # 获取当前时间\n",
"# current_time = time.strftime(\"%H:%M:%S\", time.localtime())\n",
"# current_time_1 = time.strftime(\"%H:%M:%S\", time.localtime())\n",
" # 每天定时12点运行\n",
" while True:\n",
" # 获取当前时间\n",
" current_time = time.strftime(\"%H:%M:%S\", time.localtime())\n",
" current_time_1 = time.strftime(\"%H:%M:%S\", time.localtime())\n",
" \n",
" current_time = time.strftime(\"%H:%M:%S\", time.localtime())\n",
" try:\n",
" # 判断当前时间是否为执行任务的时间点\n",
" if current_time == \"09:16:00\":\n",
" print(\"执行定时任务\")\n",
" start()\n",
" elif current_time == \"09:21:00\":\n",
" print(\"执行定时任务\")\n",
" start()\n",
" time.sleep(1)\n",
" except:\n",
" print(f\"{current_time}执行失败\")\n",
"\n",
"\n",
"# # 判断当前时间是否为执行任务的时间点\n",
"# if current_time == \"09:15:00\":\n",
"# print(\"执行定时任务\")\n",
"# queryDataListItemNos()\n",
"# start()\n",
" \n",
"# # elif current_time_1 == \"20:00:00\":\n",
"# # print(\"更新数据\")\n",
"# # start_1()\n",
"# time.sleep(1)\n",
"\n",
"\n",
"# start()\n",
"# # 检测数据准确性, 需要检测放开\n",
"# # check_data(\"100028098|LISTING_PRICE\")\n",
"# # check_data(\"9137070016544622XB|DAY_Yield\")\n"
@ -860,90 +861,25 @@
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"20250514\n",
"获取到的数据项ID['数据项编码', 'C01100047|STRIKE_PRICE', 'Brentspj', '913716251671540959|EXCHANGE_RATE', 'C01100010|LIST_PRICE01', '250326561|STRIKE_PRICE', 'C01100047|LIST_PRICE', 'C01100047|LIST_PRICE-1', 'C01100047|LIST_PRICE-01', 'OIL_CHEM|guonei|6097|PRICE', '91370500674526498A|C01100008|STRIKE_PRICE', '91370783724809024G|BEN|PRICE', '91370500737223620X|BEN|PRICE', '91370503706169019D|BEN|PRICE', '91370503164840647R|BEN|PRICE', 'C01100047|TURNOVER', '913705221649223519|C01100047|EXW', 'C01100047|CAPACITY']\n",
"获取到的数据项ID['C01100047|STRIKE_PRICE', 'Brentspj', '913716251671540959|EXCHANGE_RATE', 'C01100010|LIST_PRICE01', '250326561|STRIKE_PRICE', 'C01100047|LIST_PRICE', 'C01100047|LIST_PRICE-1', 'C01100047|LIST_PRICE-01', 'OIL_CHEM|guonei|6097|PRICE', '91370500674526498A|C01100008|STRIKE_PRICE', '91370783724809024G|BEN|PRICE', '91370500737223620X|BEN|PRICE', '91370503706169019D|BEN|PRICE', '91370503164840647R|BEN|PRICE', 'C01100047|TURNOVER', '913705221649223519|C01100047|EXW', 'C01100047|CAPACITY']\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_13740\\2721280480.py:753: DeprecationWarning:\n",
"\n",
"The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
"\n",
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_13740\\2721280480.py:302: UserWarning:\n",
"\n",
"The argument 'infer_datetime_format' is deprecated and will be removed in a future version. A strict version of it is now the default, see https://pandas.pydata.org/pdeps/0004-consistent-to-datetime-parsing.html. You can safely remove this argument.\n",
"\n",
"d:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning:\n",
"\n",
"Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 7.765e+05, tolerance: 5.324e+04\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using matplotlib backend: <object object at 0x000002AB0D800550>\n",
"%pylab is deprecated, use %matplotlib inline and import the required libraries.\n",
"Populating the interactive namespace from numpy and matplotlib\n",
"Fitting 3 folds for each of 180 candidates, totalling 540 fits\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"d:\\ProgramData\\anaconda3\\Lib\\site-packages\\IPython\\core\\magics\\pylab.py:162: UserWarning:\n",
"\n",
"pylab import has clobbered these variables: ['random', 'datetime', 'plot', '__version__']\n",
"`%matplotlib` prevents importing * from pylab and numpy\n",
"\n",
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_13740\\2721280480.py:255: UserWarning:\n",
"\n",
"The argument 'infer_datetime_format' is deprecated and will be removed in a future version. A strict version of it is now the default, see https://pandas.pydata.org/pdeps/0004-consistent-to-datetime-parsing.html. You can safely remove this argument.\n",
"\n",
"C:\\Users\\EDY\\AppData\\Local\\Temp\\ipykernel_13740\\2721280480.py:285: FutureWarning:\n",
"\n",
"Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Date\n",
"2025-05-15 6119.335449\n",
"Name: 日度预测价格, dtype: float32\n",
"{'funcModule': '数据表信息列表', 'funcOperation': '新增', 'data': [{'dataItemNo': 'C01100047|FORECAST_PRICE', 'dataDate': datetime.datetime(2025, 5, 14, 0, 0), 'dataStatus': 'add', 'dataValue': 6119.34}]}\n"
]
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"# # 自定义日期执行预测\n",
"# # # 自定义日期执行预测\n",
"\n",
"start_date = datetime(2025, 5, 14)\n",
"end_date = datetime(2025, 5, 15)\n",
"# start_date = datetime(2025, 5, 16)\n",
"# end_date = datetime(2025, 5, 19)\n",
"\n",
"token = get_head_auth()\n",
"token_push = get_head_push_auth()\n",
"while start_date < end_date:\n",
" print(start_date.strftime('%Y%m%d'))\n",
" start(start_date,token,token_push)\n",
" time.sleep(2)\n",
" # start_1(start_date)\n",
" start_date += timedelta(days=1)"
"# token = get_head_auth()\n",
"# token_push = get_head_push_auth()\n",
"# while start_date < end_date:\n",
"# print(start_date.strftime('%Y%m%d'))\n",
"# start(start_date,token,token_push)\n",
"# time.sleep(2)\n",
"# # start_1(start_date)\n",
"# start_date += timedelta(days=1)"
]
},
{
@ -956,7 +892,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "base",
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
@ -970,7 +906,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.7.0"
}
},
"nbformat": 4,

BIN
aisenzhecode/石油苯/纯苯数据项.xls
View File

Binary file not shown.

1
config_jingbo.py
View File

@ -274,6 +274,7 @@ upload_data = {
}
# 已弃用
warning_data = {
"groupNo": '', # 用户组id
"funcModule": '原油特征停更预警',

6
config_jingbo_yuedu.py
View File

@ -145,6 +145,8 @@ modelsindex = [
}
]
# eta 上传预测结果的请求体，后面发起请求的时候更改 model datalist 数据
data = {
"IndexCode": "",
@ -367,7 +369,9 @@ bdwd_items = {
'cisiyue': 'yyycbdwdcsiy',
}
# 报告中八大维度数据项重命名
columnsrename={'yyycbdwdbz': '本周', 'yyycbdwdcey': '次二月', 'yyycbdwdcr': '次日', 'yyycbdwdcsiy': '次四月',
'yyycbdwdcsy': '次三月', 'yyycbdwdcy': '次月', 'yyycbdwdcz': '次周', 'yyycbdwdgz': '隔周', }
# 北京环境数据库
host = '192.168.101.27'
port = 3306

64
config_juxiting.py
View File

@ -81,7 +81,13 @@ edbcodenamedict = {
}
# eta自有数据指标编码
modelsindex = {
bdwdname = [
'次日'
]
modelsindex = [{
'NHITS': 'SELF0000077',
'Informer': 'SELF0000078',
'LSTM': 'SELF0000079',
@ -103,23 +109,23 @@ modelsindex = {
'MLPMultivariate': 'SELF0000095',
'TiDE': 'SELF0000096',
'DeepNPTS': 'SELF0000097'
}
}]
# eta 上传预测结果的请求体，后面发起请求的时候更改 model datalist 数据
data = {
"IndexCode": "",
"IndexName": "价格预测模型",
"IndexName": "聚烯烃价格预测xx模型-yy",
"Unit": "无",
"Frequency": "日度",
"SourceName": f"价格预测",
"Remark": 'ddd',
"DataList": [
{
"Date": "2024-05-02",
"Value": 333444
}
]
"Frequency": "日度",
"SourceName": f"价格预测",
"Remark": 'ddd',
"DataList": [
{
"Date": "2024-05-02",
"Value": 333444
}
]
}
# eta 分类
@ -143,6 +149,10 @@ upload_warning_url = f"http://{server_host}:8080/jingbo-dev/api/basicBuiness/cru
query_data_list_item_nos_url = f"http://{server_host}:8080/jingbo-dev/api/warehouse/dwDataItem/queryDataListItemNos"
# 上传数据项值
push_data_value_list_url = f"http://{server_host}:8080/jingbo-dev/api/dw/dataValue/pushDataValueList"
# 上传停更数据到市场信息平台
push_waring_data_value_list_url = f"http://{server_host}:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/crudeSaveOrupdate"
# 获取预警数据中取消订阅指标ID
get_waring_data_value_list_url = f"http://{server_host}:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/dataList"
login_data = {
"data": {
@ -173,7 +183,7 @@ upload_data = {
}
}
# 已弃用
warning_data = {
"funcModule": '原油特征停更预警',
"funcOperation": '原油特征停更预警',
@ -219,6 +229,30 @@ push_data_value_list_data = {
}
push_waring_data_value_list_data = {
"data": {
"crudeOilWarningDtoList": [
{
"lastUpdateDate": "20240501",
"updateSuspensionCycle": 1,
"dataSource": "9",
"frequency": "1",
"indicatorName": "美元指数",
"indicatorId": "myzs001",
"warningDate": "2024-05-13"
}
],
"dataSource": "9"
},
"funcModule": "商品数据同步",
"funcOperation": "同步"
}
get_waring_data_value_list_data = {
"data": "9", "funcModule": "商品数据同步", "funcOperation": "同步"}
# 八大维度数据项编码
bdwd_items = {
'ciri': 'jxtppbdwdcr',
@ -227,7 +261,7 @@ bdwd_items = {
'gezhou': 'jxtppbdwdgz',
'ciyue': 'jxtppbdwdcy',
'cieryue': 'jxtppbdwdcey',
'cisanyue': 'jxtppbdwdcsy',
'cisanyue': 'jxtppbdwdcsany',
'cisiyue': 'jxtppbdwdcsiy',
}
# 北京环境数据库
@ -250,7 +284,7 @@ is_edbcode = False # 特征使用edbcoding列表中的
is_edbnamelist = False # 自定义特征，对应上面的edbnamelist
is_update_eta = True # 预测结果上传到eta
is_update_report = True # 是否上传报告
is_update_warning_data = False # 是否上传预警数据
is_update_warning_data = True # 是否上传预警数据
is_del_corr = 0.6 # 是否删除相关性高的特征,取值为 0-1 ，0 为不删除，0.6 表示删除相关性小于0.6的特征
is_del_tow_month = True # 是否删除两个月不更新的特征

293
config_juxiting_yuedu.py
View File

@ -80,36 +80,114 @@ edbcodenamedict = {
}
# eta自有数据指标编码
modelsindex = {
'NHITS': 'SELF0000077',
'Informer': 'SELF0000078',
'LSTM': 'SELF0000079',
'iTransformer': 'SELF0000080',
'TSMixer': 'SELF0000081',
'TSMixerx': 'SELF0000082',
'PatchTST': 'SELF0000083',
'RNN': 'SELF0000084',
'GRU': 'SELF0000085',
'TCN': 'SELF0000086',
'BiTCN': 'SELF0000087',
'DilatedRNN': 'SELF0000088',
'MLP': 'SELF0000089',
'DLinear': 'SELF0000090',
'NLinear': 'SELF0000091',
'TFT': 'SELF0000092',
'FEDformer': 'SELF0000093',
'StemGNN': 'SELF0000094',
'MLPMultivariate': 'SELF0000095',
'TiDE': 'SELF0000096',
'DeepNPTS': 'SELF0000097'
}
# eta自有数据指标编码 次月,次二月，次三月，次四月
bdwdname = [
'次月',
'次二月',
'次三月',
'次四月',
]
modelsindex = [
{
"NHITS": "SELF0000275",
"Informer": "SELF0000276",
"LSTM": "SELF0000277",
"iTransformer": "SELF0000278",
"TSMixer": "SELF0000279",
"TSMixerx": "SELF0000280",
"PatchTST": "SELF0000281",
"RNN": "SELF0000282",
"GRU": "SELF0000283",
"TCN": "SELF0000284",
"BiTCN": "SELF0000285",
"DilatedRNN": "SELF0000286",
"MLP": "SELF0000287",
"DLinear": "SELF0000288",
"NLinear": "SELF0000289",
"TFT": "SELF0000290",
"FEDformer": "SELF0000291",
"StemGNN": "SELF0000292",
"MLPMultivariate": "SELF0000293",
"TiDE": "SELF0000294",
"DeepNPTS": "SELF0000295",
"NBEATS": "SELF0000296", },
{
"NHITS": "SELF0000297",
"Informer": "SELF0000298",
"LSTM": "SELF0000299",
"iTransformer": "SELF0000300",
"TSMixer": "SELF0000301",
"TSMixerx": "SELF0000302",
"PatchTST": "SELF0000303",
"RNN": "SELF0000304",
"GRU": "SELF0000305",
"TCN": "SELF0000306",
"BiTCN": "SELF0000307",
"DilatedRNN": "SELF0000308",
"MLP": "SELF0000309",
"DLinear": "SELF0000310",
"NLinear": "SELF0000311",
"TFT": "SELF0000312",
"FEDformer": "SELF0000313",
"StemGNN": "SELF0000314",
"MLPMultivariate": "SELF0000315",
"TiDE": "SELF0000316",
"DeepNPTS": "SELF0000317",
"NBEATS": "SELF0000318", },
{
"NHITS": "SELF0000319",
"Informer": "SELF0000320",
"LSTM": "SELF0000321",
"iTransformer": "SELF0000322",
"TSMixer": "SELF0000323",
"TSMixerx": "SELF0000324",
"PatchTST": "SELF0000325",
"RNN": "SELF0000326",
"GRU": "SELF0000327",
"TCN": "SELF0000328",
"BiTCN": "SELF0000329",
"DilatedRNN": "SELF0000330",
"MLP": "SELF0000331",
"DLinear": "SELF0000332",
"NLinear": "SELF0000333",
"TFT": "SELF0000334",
"FEDformer": "SELF0000335",
"StemGNN": "SELF0000336",
"MLPMultivariate": "SELF0000337",
"TiDE": "SELF0000338",
"DeepNPTS": "SELF0000339",
"NBEATS": "SELF0000340", },
{
"NHITS": "SELF0000341",
"Informer": "SELF0000342",
"LSTM": "SELF0000343",
"iTransformer": "SELF0000344",
"TSMixer": "SELF0000345",
"TSMixerx": "SELF0000346",
"PatchTST": "SELF0000347",
"RNN": "SELF0000348",
"GRU": "SELF0000349",
"TCN": "SELF0000350",
"BiTCN": "SELF0000351",
"DilatedRNN": "SELF0000352",
"MLP": "SELF0000353",
"DLinear": "SELF0000354",
"NLinear": "SELF0000355",
"TFT": "SELF0000356",
"FEDformer": "SELF0000357",
"StemGNN": "SELF0000358",
"MLPMultivariate": "SELF0000359",
"TiDE": "SELF0000360",
"DeepNPTS": "SELF0000361",
"NBEATS": "SELF0000362",
}]
# eta 上传预测结果的请求体，后面发起请求的时候更改 model datalist 数据
data = {
"IndexCode": "",
"IndexName": "价格预测模型",
"IndexName": "聚烯烃价格预测xx模型-yy",
"Unit": "无",
"Frequency": "日度",
"SourceName": f"价格预测",
@ -131,13 +209,118 @@ data = {
ClassifyId = 1161
# 变量定义--测试环境
server_host = '192.168.100.53'
# 变量定义--线上环境
# server_host = '10.200.32.39'
# login_pushreport_url = "http://10.200.32.39/jingbo-api/api/server/login"
# upload_url = "http://10.200.32.39/jingbo-api/api/analysis/reportInfo/researchUploadReportSave"
# upload_warning_url = "http://10.200.32.39/jingbo-api/api/basicBuiness/crudeOilWarning/save"
# query_data_list_item_nos_url = f"http://{server_host}/jingbo-api/api/warehouse/dwDataItem/queryDataListItemNos"
# # 上传数据项值
# push_data_value_list_url = f"http://{server_host}/jingbo-api/api/dw/dataValue/pushDataValueList"
login_pushreport_url = f"http://{server_host}:8080/jingbo-dev/api/server/login"
upload_url = f"http://{server_host}:8080/jingbo-dev/api/analysis/reportInfo/researchUploadReportSave"
upload_warning_url = f"http://{server_host}:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/save"
query_data_list_item_nos_url = f"http://{server_host}:8080/jingbo-dev/api/warehouse/dwDataItem/queryDataListItemNos"
# login_data = {
# "data": {
# "account": "api_dev",
# "password": "ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=",
# "tenantHashCode": "8a4577dbd919675758d57999a1e891fe",
# "terminal": "API"
# },
# "funcModule": "API",
# "funcOperation": "获取token"
# }
# upload_data = {
# "funcModule":'研究报告信息',
# "funcOperation":'上传原油价格预测报告',
# "data":{
# "groupNo":'', # 用户组id
# "ownerAccount":'27663', #报告所属用户账号 27663 - 刘小朋
# "reportType":'OIL_PRICE_FORECAST', # 报告类型，固定为OIL_PRICE_FORECAST
# "fileName": '', #文件名称
# "fileBase64": '' ,#文件内容base64
# "categoryNo":'yyjgycbg', # 研究报告分类编码
# "smartBusinessClassCode":'YCJGYCBG', #分析报告分类编码
# "reportEmployeeCode":"E40482" ,# 报告人 E40482 - 管理员 0000027663 - 刘小朋
# "reportDeptCode" :"002000621000", # 报告部门 - 002000621000 SH期货研究部
# "productGroupCode":"RAW_MATERIAL" # 商品分类
# }
# }
# warning_data = {
# "groupNo":'', # 用户组id
# "funcModule":'原油特征停更预警',
# "funcOperation":'原油特征停更预警',
# "data":{
# 'WARNING_TYPE_NAME':'特征数据停更预警',
# 'WARNING_CONTENT':'',
# 'WARNING_DATE':''
# }
# }
# query_data_list_item_nos_data = {
# "funcModule": "数据项",
# "funcOperation": "查询",
# "data": {
# "dateStart":"20200101",
# "dateEnd":"20241231",
# "dataItemNoList":["Brentzdj","Brentzgj"] # 数据项编码,代表 brent最低价和最高价
# }
# }
# push_data_value_list_data = {
# "funcModule": "数据表信息列表",
# "funcOperation": "新增",
# "data": [
# {"dataItemNo": "91230600716676129",
# "dataDate": "20230113",
# "dataStatus": "add",
# "dataValue": 100.11
# },
# {"dataItemNo": "91230600716676129P|ETHYL_BEN|CAPACITY",
# "dataDate": "20230113",
# "dataStatus": "add",
# "dataValue": 100.55
# },
# {"dataItemNo": "91230600716676129P|ETHYL_BEN|CAPACITY",
# "dataDate": "20230113",
# "dataStatus": "add",
# "dataValue": 100.55
# }
# ]
# }
# # 八大维度数据项编码
# bdwd_items = {
# 'ciri': '原油大数据预测|FORECAST|PRICE|T',
# 'benzhou': '原油大数据预测|FORECAST|PRICE|W',
# 'cizhou': '原油大数据预测|FORECAST|PRICE|W_1',
# 'gezhou': '原油大数据预测|FORECAST|PRICE|W_2',
# 'ciyue': '原油大数据预测|FORECAST|PRICE|M_1',
# 'cieryue': '原油大数据预测|FORECAST|PRICE|M_2',
# 'cisanyue': '原油大数据预测|FORECAST|PRICE|M_3',
# 'cisiyue': '原油大数据预测|FORECAST|PRICE|M_4',
# }
# # 生产环境数据库
# host = 'rm-2zehj3r1n60ttz9x5.mysql.rds.aliyuncs.com'
# port = 3306
# dbusername ='jingbo'
# password = 'shihua@123'
# dbname = 'jingbo'
# table_name = 'v_tbl_crude_oil_warning'
# # 变量定义--测试环境
server_host = '192.168.100.53:8080' # 内网
# server_host = '183.242.74.28' # 外网
login_pushreport_url = f"http://{server_host}/jingbo-dev/api/server/login"
upload_url = f"http://{server_host}/jingbo-dev/api/analysis/reportInfo/researchUploadReportSave"
upload_warning_url = f"http://{server_host}/jingbo-dev/api/basicBuiness/crudeOilWarning/save"
query_data_list_item_nos_url = f"http://{server_host}/jingbo-dev/api/warehouse/dwDataItem/queryDataListItemNos"
# 上传数据项值
push_data_value_list_url = f"http://{server_host}/jingbo-dev/api/dw/dataValue/pushDataValueList"
login_data = {
"data": {
@ -188,6 +371,42 @@ query_data_list_item_nos_data = {
}
}
push_data_value_list_data = {
"funcModule": "数据表信息列表",
"funcOperation": "新增",
"data": [
{"dataItemNo": "91230600716676129",
"dataDate": "20230113",
"dataStatus": "add",
"dataValue": 100.11
},
{"dataItemNo": "91230600716676129P|ETHYL_BEN|CAPACITY",
"dataDate": "20230113",
"dataStatus": "add",
"dataValue": 100.55
},
{"dataItemNo": "91230600716676129P|ETHYL_BEN|CAPACITY",
"dataDate": "20230113",
"dataStatus": "add",
"dataValue": 100.55
}
]
}
# 八大维度数据项编码
bdwd_items = {
'ciri': 'jxtppbdwdcr',
'benzhou': 'jxtppbdwdbz',
'cizhou': 'jxtppbdwdcz',
'gezhou': 'jxtppbdwdgz',
'ciyue': 'jxtppbdwdcy',
'cieryue': 'jxtppbdwdcey',
'cisanyue': 'jxtppbdwdcsany',
'cisiyue': 'jxtppbdwdcsiy',
}
# 报告中八大维度数据项重命名
columnsrename={'jxtppbdwdbz': '本周', 'jxtppbdwdcey': '次二月', 'jxtppbdwdcr': '次日', 'jxtppbdwdcsiy': '次四月',
'jxtppbdwdcsany': '次三月', 'jxtppbdwdcy': '次月', 'jxtppbdwdcz': '次周', 'jxtppbdwdgz': '隔周', }
# 北京环境数据库
host = '192.168.101.27'
@ -207,11 +426,13 @@ is_timefurture = True # 是否使用时间特征
is_fivemodels = False # 是否使用之前保存的最佳的5个模型
is_edbcode = False # 特征使用edbcoding列表中的
is_edbnamelist = False # 自定义特征，对应上面的edbnamelist
is_update_eta = False # 预测结果上传到eta
is_update_report = False # 是否上传报告
is_update_eta = True # 预测结果上传到eta
is_update_report = True # 是否上传报告
is_update_warning_data = False # 是否上传预警数据
is_del_corr = 0.6 # 是否删除相关性高的特征,取值为 0-1 ，0 为不删除，0.6 表示删除相关性小于0.6的特征
is_del_tow_month = True # 是否删除两个月不更新的特征
is_update_predict_value = True # 是否上传预测值到市场信息平台
is_del_corr = 0 # 是否删除相关性高的特征,取值为 0-1 ，0 为不删除，0.6 表示删除相关性小于0.6的特征
is_del_tow_month = False # 是否删除两个月不更新的特征
is_bdwd = True # 是否使用八大维度
# 连接到数据库
@ -247,7 +468,7 @@ train_steps = 50 if is_debug else 1000 # 训练步数,用来限定epoch次数
val_check_steps = 30 # 评估频率
early_stop_patience_steps = 5 # 早停的耐心步数
# --- 交叉验证用的参数
test_size = 200 # 测试集大小，定义100，后面使用的时候重新赋值
test_size = 100 # 测试集大小，定义100，后面使用的时候重新赋值
val_size = test_size # 验证集大小,同测试集大小
# 特征筛选用到的参数

140
config_juxiting_zhoudu.py
View File

@ -80,36 +80,64 @@ edbcodenamedict = {
}
# eta自有数据指标编码
modelsindex = {
'NHITS': 'SELF0000077',
'Informer': 'SELF0000078',
'LSTM': 'SELF0000079',
'iTransformer': 'SELF0000080',
'TSMixer': 'SELF0000081',
'TSMixerx': 'SELF0000082',
'PatchTST': 'SELF0000083',
'RNN': 'SELF0000084',
'GRU': 'SELF0000085',
'TCN': 'SELF0000086',
'BiTCN': 'SELF0000087',
'DilatedRNN': 'SELF0000088',
'MLP': 'SELF0000089',
'DLinear': 'SELF0000090',
'NLinear': 'SELF0000091',
'TFT': 'SELF0000092',
'FEDformer': 'SELF0000093',
'StemGNN': 'SELF0000094',
'MLPMultivariate': 'SELF0000095',
'TiDE': 'SELF0000096',
'DeepNPTS': 'SELF0000097'
}
# eta自有数据指标编码 次周,隔周
bdwdname = [
'次周',
'隔周',
]
modelsindex = [{
"NHITS": "SELF0000231",
"Informer": "SELF0000232",
"LSTM": "SELF0000233",
"iTransformer": "SELF0000234",
"TSMixer": "SELF0000235",
"TSMixerx": "SELF0000236",
"PatchTST": "SELF0000237",
"RNN": "SELF0000238",
"GRU": "SELF0000239",
"TCN": "SELF0000240",
"BiTCN": "SELF0000241",
"DilatedRNN": "SELF0000242",
"MLP": "SELF0000243",
"DLinear": "SELF0000244",
"NLinear": "SELF0000245",
"TFT": "SELF0000246",
"FEDformer": "SELF0000247",
"StemGNN": "SELF0000248",
"MLPMultivariate": "SELF0000249",
"TiDE": "SELF0000250",
"DeepNPTS": "SELF0000251",
"NBEATS": "SELF0000252",
},
{
"NHITS": "SELF0000253",
"Informer": "SELF0000254",
"LSTM": "SELF0000255",
"iTransformer": "SELF0000256",
"TSMixer": "SELF0000257",
"TSMixerx": "SELF0000258",
"PatchTST": "SELF0000259",
"RNN": "SELF0000260",
"GRU": "SELF0000261",
"TCN": "SELF0000262",
"BiTCN": "SELF0000263",
"DilatedRNN": "SELF0000264",
"MLP": "SELF0000265",
"DLinear": "SELF0000266",
"NLinear": "SELF0000267",
"TFT": "SELF0000268",
"FEDformer": "SELF0000269",
"StemGNN": "SELF0000270",
"MLPMultivariate": "SELF0000271",
"TiDE": "SELF0000272",
"DeepNPTS": "SELF0000273",
"NBEATS": "SELF0000274",
}]
# eta 上传预测结果的请求体，后面发起请求的时候更改 model datalist 数据
data = {
"IndexCode": "",
"IndexName": "价格预测模型",
"IndexName": "聚烯烃PP价格预测xx模型-yy",
"Unit": "无",
"Frequency": "日度",
"SourceName": f"价格预测",
@ -132,12 +160,15 @@ ClassifyId = 1161
# 变量定义--测试环境
server_host = '192.168.100.53'
server_host = '192.168.100.53:8080' # 内网
# server_host = '183.242.74.28' # 外网
login_pushreport_url = f"http://{server_host}:8080/jingbo-dev/api/server/login"
upload_url = f"http://{server_host}:8080/jingbo-dev/api/analysis/reportInfo/researchUploadReportSave"
upload_warning_url = f"http://{server_host}:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/save"
query_data_list_item_nos_url = f"http://{server_host}:8080/jingbo-dev/api/warehouse/dwDataItem/queryDataListItemNos"
login_pushreport_url = f"http://{server_host}/jingbo-dev/api/server/login"
upload_url = f"http://{server_host}/jingbo-dev/api/analysis/reportInfo/researchUploadReportSave"
upload_warning_url = f"http://{server_host}/jingbo-dev/api/basicBuiness/crudeOilWarning/save"
query_data_list_item_nos_url = f"http://{server_host}/jingbo-dev/api/warehouse/dwDataItem/queryDataListItemNos"
# 上传数据项值
push_data_value_list_url = f"http://{server_host}/jingbo-dev/api/dw/dataValue/pushDataValueList"
login_data = {
"data": {
@ -188,6 +219,40 @@ query_data_list_item_nos_data = {
}
}
push_data_value_list_data = {
"funcModule": "数据表信息列表",
"funcOperation": "新增",
"data": [
{"dataItemNo": "91230600716676129",
"dataDate": "20230113",
"dataStatus": "add",
"dataValue": 100.11
},
{"dataItemNo": "91230600716676129P|ETHYL_BEN|CAPACITY",
"dataDate": "20230113",
"dataStatus": "add",
"dataValue": 100.55
},
{"dataItemNo": "91230600716676129P|ETHYL_BEN|CAPACITY",
"dataDate": "20230113",
"dataStatus": "add",
"dataValue": 100.55
}
]
}
# 八大维度数据项编码
bdwd_items = {
'ciri': 'jxtppbdwdcr',
'benzhou': 'jxtppbdwdbz',
'cizhou': 'jxtppbdwdcz',
'gezhou': 'jxtppbdwdgz',
'ciyue': 'jxtppbdwdcy',
'cieryue': 'jxtppbdwdcey',
'cisanyue': 'jxtppbdwdcsany',
'cisiyue': 'jxtppbdwdcsiy',
}
# 北京环境数据库
host = '192.168.101.27'
@ -207,11 +272,13 @@ is_timefurture = True # 是否使用时间特征
is_fivemodels = False # 是否使用之前保存的最佳的5个模型
is_edbcode = False # 特征使用edbcoding列表中的
is_edbnamelist = False # 自定义特征，对应上面的edbnamelist
is_update_eta = False # 预测结果上传到eta
is_update_report = False # 是否上传报告
is_update_eta = True # 预测结果上传到eta
is_update_report = True # 是否上传报告
is_update_warning_data = False # 是否上传预警数据
is_update_predict_value = True # 是否上传预测值到市场信息平台
is_del_corr = 0.6 # 是否删除相关性高的特征,取值为 0-1 ，0 为不删除，0.6 表示删除相关性小于0.6的特征
is_del_tow_month = True # 是否删除两个月不更新的特征
is_del_tow_month = False # 是否删除两个月不更新的特征
is_bdwd = False # 是否使用八大维度
# 连接到数据库
@ -224,7 +291,7 @@ print("数据库连接成功", host, dbname, dbusername)
# 数据截取日期
start_year = 2015 # 数据开始年份
end_time = '' # 数据截取日期
freq = 'WW' # 时间频率，"D": 天 "W": 周"M": 月"Q": 季度"A": 年 "H": 小时 "T": 分钟 "S": 秒 "B": 工作日
freq = 'WW' # 时间频率，"D": 天 "W": 周"M": 月"Q": 季度"A": 年 "H": 小时 "T": 分钟 "S": 秒 "B": 工作日 "WW" 自定义周
delweekenday = True if freq == 'B' else False # 是否删除周末数据
is_corr = False # 特征是否参与滞后领先提升相关系数
add_kdj = False # 是否添加kdj指标
@ -263,6 +330,9 @@ weight_dict = [0.4, 0.15, 0.1, 0.1, 0.25] # 权重
data_set = 'PP指标数据.xlsx' # 数据集文件
dataset = 'juxitingzhoududataset' # 数据集文件夹
print("当前工作目录:", os.getcwd())
print("数据库路径:", os.path.abspath('juxitingzhoududataset/jbsh_juxiting_zhoudu.db'))
# 数据库名称
db_name = os.path.join(dataset, 'jbsh_juxiting_zhoudu.db')
sqlitedb = SQLiteHandler(db_name)

241
eta数据调试.py
View File

@ -3,6 +3,92 @@
from config_jingbo import *
from lib.dataread import *
global_config.update({
# 核心参数
'logger': logger,
'dataset': dataset,
'y': y,
'is_debug': is_debug,
'is_train': is_train,
'is_fivemodels': is_fivemodels,
'is_update_report': is_update_report,
'settings': settings,
'weight_dict': weight_dict,
'bdwdname': bdwdname,
# 模型参数
'data_set': data_set,
'input_size': input_size,
'horizon': horizon,
'train_steps': train_steps,
'val_check_steps': val_check_steps,
'val_size': val_size,
'test_size': test_size,
'modelsindex': modelsindex,
'rote': rote,
'bdwd_items': bdwd_items,
# 特征工程开关
'is_del_corr': is_del_corr,
'is_del_tow_month': is_del_tow_month,
'is_eta': is_eta,
'is_update_eta': is_update_eta,
'is_fivemodels': is_fivemodels,
'is_update_predict_value': is_update_predict_value,
'early_stop_patience_steps': early_stop_patience_steps,
# 时间参数
'start_year': start_year,
'end_time': end_time or datetime.datetime.now().strftime("%Y-%m-%d"),
'freq': freq, # 保持列表结构
# 接口配置
'login_pushreport_url': login_pushreport_url,
'login_data': login_data,
'upload_url': upload_url,
'upload_data': upload_data,
'upload_warning_url': upload_warning_url,
'warning_data': warning_data,
# 查询接口
'query_data_list_item_nos_url': query_data_list_item_nos_url,
'query_data_list_item_nos_data': query_data_list_item_nos_data,
# 上传数据项
'push_data_value_list_url': push_data_value_list_url,
'push_data_value_list_data': push_data_value_list_data,
# 上传预警数据
'push_waring_data_value_list_url': push_waring_data_value_list_url,
'push_waring_data_value_list_data': push_waring_data_value_list_data,
# 获取取消订阅的数据
'get_waring_data_value_list_url': get_waring_data_value_list_url,
'get_waring_data_value_list_data': get_waring_data_value_list_data,
# eta 配置
'APPID': APPID,
'SECRET': SECRET,
'etadata': data,
'edbcodelist': edbcodelist,
'ClassifyId': ClassifyId,
'edbcodedataurl': edbcodedataurl,
'classifyidlisturl': classifyidlisturl,
'edbdatapushurl': edbdatapushurl,
'edbdeleteurl': edbdeleteurl,
'edbbusinessurl': edbbusinessurl,
'ClassifyId': ClassifyId,
'classifylisturl': classifylisturl,
# 数据库配置
'sqlitedb': sqlitedb,
'bdwd_items': bdwd_items,
'is_bdwd': is_bdwd,
})
logger.info('从eta获取数据...')
signature = BinanceAPI(APPID, SECRET)
etadata = EtaReader(signature=signature,
@ -13,7 +99,8 @@ etadata = EtaReader(signature=signature,
edbdatapushurl=edbdatapushurl,
edbdeleteurl=edbdeleteurl,
edbbusinessurl=edbbusinessurl,
)
classifyId=ClassifyId,)
# eta自有数据指标编码
modelsindex = {
@ -40,16 +127,152 @@ modelsindex = {
'DeepNPTS': 'SELF0000097'
}
selfid = {
"聚烯烃价格预测NHITS模型-次周": "SELF0000231",
"聚烯烃价格预测Informer模型-次周": "SELF0000232",
"聚烯烃价格预测LSTM模型-次周": "SELF0000233",
"聚烯烃价格预测iTransformer模型-次周": "SELF0000234",
"聚烯烃价格预测TSMixer模型-次周": "SELF0000235",
"聚烯烃价格预测TSMixerx模型-次周": "SELF0000236",
"聚烯烃价格预测PatchTST模型-次周": "SELF0000237",
"聚烯烃价格预测RNN模型-次周": "SELF0000238",
"聚烯烃价格预测GRU模型-次周": "SELF0000239",
"聚烯烃价格预测TCN模型-次周": "SELF0000240",
"聚烯烃价格预测BiTCN模型-次周": "SELF0000241",
"聚烯烃价格预测DilatedRNN模型-次周": "SELF0000242",
"聚烯烃价格预测MLP模型-次周": "SELF0000243",
"聚烯烃价格预测DLinear模型-次周": "SELF0000244",
"聚烯烃价格预测NLinear模型-次周": "SELF0000245",
"聚烯烃价格预测TFT模型-次周": "SELF0000246",
"聚烯烃价格预测FEDformer模型-次周": "SELF0000247",
"聚烯烃价格预测StemGNN模型-次周": "SELF0000248",
"聚烯烃价格预测MLPMultivariate模型-次周": "SELF0000249",
"聚烯烃价格预测TiDE模型-次周": "SELF0000250",
"聚烯烃价格预测DeepNPTS模型-次周": "SELF0000251",
"聚烯烃价格预测NBEATS模型-次周": "SELF0000252",
"聚烯烃价格预测NHITS模型-隔周": "SELF0000253",
"聚烯烃价格预测Informer模型-隔周": "SELF0000254",
"聚烯烃价格预测LSTM模型-隔周": "SELF0000255",
"聚烯烃价格预测iTransformer模型-隔周": "SELF0000256",
"聚烯烃价格预测TSMixer模型-隔周": "SELF0000257",
"聚烯烃价格预测TSMixerx模型-隔周": "SELF0000258",
"聚烯烃价格预测PatchTST模型-隔周": "SELF0000259",
"聚烯烃价格预测RNN模型-隔周": "SELF0000260",
"聚烯烃价格预测GRU模型-隔周": "SELF0000261",
"聚烯烃价格预测TCN模型-隔周": "SELF0000262",
"聚烯烃价格预测BiTCN模型-隔周": "SELF0000263",
"聚烯烃价格预测DilatedRNN模型-隔周": "SELF0000264",
"聚烯烃价格预测MLP模型-隔周": "SELF0000265",
"聚烯烃价格预测DLinear模型-隔周": "SELF0000266",
"聚烯烃价格预测NLinear模型-隔周": "SELF0000267",
"聚烯烃价格预测TFT模型-隔周": "SELF0000268",
"聚烯烃价格预测FEDformer模型-隔周": "SELF0000269",
"聚烯烃价格预测StemGNN模型-隔周": "SELF0000270",
"聚烯烃价格预测MLPMultivariate模型-隔周": "SELF0000271",
"聚烯烃价格预测TiDE模型-隔周": "SELF0000272",
"聚烯烃价格预测DeepNPTS模型-隔周": "SELF0000273",
"聚烯烃价格预测NBEATS模型-隔周": "SELF0000274",
"聚烯烃价格预测NHITS模型-次月": "SELF0000275",
"聚烯烃价格预测Informer模型-次月": "SELF0000276",
"聚烯烃价格预测LSTM模型-次月": "SELF0000277",
"聚烯烃价格预测iTransformer模型-次月": "SELF0000278",
"聚烯烃价格预测TSMixer模型-次月": "SELF0000279",
"聚烯烃价格预测TSMixerx模型-次月": "SELF0000280",
"聚烯烃价格预测PatchTST模型-次月": "SELF0000281",
"聚烯烃价格预测RNN模型-次月": "SELF0000282",
"聚烯烃价格预测GRU模型-次月": "SELF0000283",
"聚烯烃价格预测TCN模型-次月": "SELF0000284",
"聚烯烃价格预测BiTCN模型-次月": "SELF0000285",
"聚烯烃价格预测DilatedRNN模型-次月": "SELF0000286",
"聚烯烃价格预测MLP模型-次月": "SELF0000287",
"聚烯烃价格预测DLinear模型-次月": "SELF0000288",
"聚烯烃价格预测NLinear模型-次月": "SELF0000289",
"聚烯烃价格预测TFT模型-次月": "SELF0000290",
"聚烯烃价格预测FEDformer模型-次月": "SELF0000291",
"聚烯烃价格预测StemGNN模型-次月": "SELF0000292",
"聚烯烃价格预测MLPMultivariate模型-次月": "SELF0000293",
"聚烯烃价格预测TiDE模型-次月": "SELF0000294",
"聚烯烃价格预测DeepNPTS模型-次月": "SELF0000295",
"聚烯烃价格预测NBEATS模型-次月": "SELF0000296",
"聚烯烃价格预测NHITS模型-次二月": "SELF0000297",
"聚烯烃价格预测Informer模型-次二月": "SELF0000298",
"聚烯烃价格预测LSTM模型-次二月": "SELF0000299",
"聚烯烃价格预测iTransformer模型-次二月": "SELF0000300",
"聚烯烃价格预测TSMixer模型-次二月": "SELF0000301",
"聚烯烃价格预测TSMixerx模型-次二月": "SELF0000302",
"聚烯烃价格预测PatchTST模型-次二月": "SELF0000303",
"聚烯烃价格预测RNN模型-次二月": "SELF0000304",
"聚烯烃价格预测GRU模型-次二月": "SELF0000305",
"聚烯烃价格预测TCN模型-次二月": "SELF0000306",
"聚烯烃价格预测BiTCN模型-次二月": "SELF0000307",
"聚烯烃价格预测DilatedRNN模型-次二月": "SELF0000308",
"聚烯烃价格预测MLP模型-次二月": "SELF0000309",
"聚烯烃价格预测DLinear模型-次二月": "SELF0000310",
"聚烯烃价格预测NLinear模型-次二月": "SELF0000311",
"聚烯烃价格预测TFT模型-次二月": "SELF0000312",
"聚烯烃价格预测FEDformer模型-次二月": "SELF0000313",
"聚烯烃价格预测StemGNN模型-次二月": "SELF0000314",
"聚烯烃价格预测MLPMultivariate模型-次二月": "SELF0000315",
"聚烯烃价格预测TiDE模型-次二月": "SELF0000316",
"聚烯烃价格预测DeepNPTS模型-次二月": "SELF0000317",
"聚烯烃价格预测NBEATS模型-次二月": "SELF0000318",
"聚烯烃价格预测NHITS模型-次三月": "SELF0000319",
"聚烯烃价格预测Informer模型-次三月": "SELF0000320",
"聚烯烃价格预测LSTM模型-次三月": "SELF0000321",
"聚烯烃价格预测iTransformer模型-次三月": "SELF0000322",
"聚烯烃价格预测TSMixer模型-次三月": "SELF0000323",
"聚烯烃价格预测TSMixerx模型-次三月": "SELF0000324",
"聚烯烃价格预测PatchTST模型-次三月": "SELF0000325",
"聚烯烃价格预测RNN模型-次三月": "SELF0000326",
"聚烯烃价格预测GRU模型-次三月": "SELF0000327",
"聚烯烃价格预测TCN模型-次三月": "SELF0000328",
"聚烯烃价格预测BiTCN模型-次三月": "SELF0000329",
"聚烯烃价格预测DilatedRNN模型-次三月": "SELF0000330",
"聚烯烃价格预测MLP模型-次三月": "SELF0000331",
"聚烯烃价格预测DLinear模型-次三月": "SELF0000332",
"聚烯烃价格预测NLinear模型-次三月": "SELF0000333",
"聚烯烃价格预测TFT模型-次三月": "SELF0000334",
"聚烯烃价格预测FEDformer模型-次三月": "SELF0000335",
"聚烯烃价格预测StemGNN模型-次三月": "SELF0000336",
"聚烯烃价格预测MLPMultivariate模型-次三月": "SELF0000337",
"聚烯烃价格预测TiDE模型-次三月": "SELF0000338",
"聚烯烃价格预测DeepNPTS模型-次三月": "SELF0000339",
"聚烯烃价格预测NBEATS模型-次三月": "SELF0000340",
"聚烯烃价格预测NHITS模型-次四月": "SELF0000341",
"聚烯烃价格预测Informer模型-次四月": "SELF0000342",
"聚烯烃价格预测LSTM模型-次四月": "SELF0000343",
"聚烯烃价格预测iTransformer模型-次四月": "SELF0000344",
"聚烯烃价格预测TSMixer模型-次四月": "SELF0000345",
"聚烯烃价格预测TSMixerx模型-次四月": "SELF0000346",
"聚烯烃价格预测PatchTST模型-次四月": "SELF0000347",
"聚烯烃价格预测RNN模型-次四月": "SELF0000348",
"聚烯烃价格预测GRU模型-次四月": "SELF0000349",
"聚烯烃价格预测TCN模型-次四月": "SELF0000350",
"聚烯烃价格预测BiTCN模型-次四月": "SELF0000351",
"聚烯烃价格预测DilatedRNN模型-次四月": "SELF0000352",
"聚烯烃价格预测MLP模型-次四月": "SELF0000353",
"聚烯烃价格预测DLinear模型-次四月": "SELF0000354",
"聚烯烃价格预测NLinear模型-次四月": "SELF0000355",
"聚烯烃价格预测TFT模型-次四月": "SELF0000356",
"聚烯烃价格预测FEDformer模型-次四月": "SELF0000357",
"聚烯烃价格预测StemGNN模型-次四月": "SELF0000358",
"聚烯烃价格预测MLPMultivariate模型-次四月": "SELF0000359",
"聚烯烃价格预测TiDE模型-次四月": "SELF0000360",
"聚烯烃价格预测DeepNPTS模型-次四月": "SELF0000361",
"聚烯烃价格预测NBEATS模型-次四月": "SELF0000362",
}
date = '2025-02-13'
value = 333444
for m in modelsindex.keys():
# list = []
# list.append({'Date':date,'Value':value})
# data['DataList'] = list
# data['IndexCode'] = modelsindex[m]
# data['IndexName'] = f'聚烯烃价格预测{m}模型'
# data['Remark'] = m
# etadata.push_data(data)
for k, v in selfid.items():
list = []
list.append({'Date': date, 'Value': value})
data['DataList'] = list
data['IndexCode'] = v
data['IndexName'] = k
data['Remark'] = k
etadata.push_data(data)
# data['IndexCode'] = modelsindex[m]
# data['StartDate'] = '2025-01-01'

115
lib/dataread.py
View File

@ -995,14 +995,13 @@ def datachuli_juxiting(df_zhibiaoshuju, df_zhibiaoliebiao, datecol='date', end_t
# 按时间顺序排列
df.sort_values(by='ds', inplace=True)
df['ds'] = pd.to_datetime(df['ds'])
# 获取2018年到当前日期的数据
df = df[df['ds'].dt.year >= 2018]
# 获取小于等于当前日期的数据
# 获取start_year年到end_time的数据
df = df[df['ds'].dt.year >= config.start_year]
df = df[df['ds'] <= end_time]
config.logger.info(f'删除两月不更新特征前数据量：{df.shape}')
# 去掉近最后数据对应的日期在两月以前的列，删除近2月的数据是常数的列
current_date = datetime.datetime.now()
two_months_ago = current_date - timedelta(days=40)
# current_date = datetime.datetime.now()
# two_months_ago = current_date - timedelta(days=40)
# 检查两月不更新的特征
# def check_column(col_name):
@ -1021,9 +1020,28 @@ def datachuli_juxiting(df_zhibiaoshuju, df_zhibiaoliebiao, datecol='date', end_t
# config.logger.info(f'删除两月不更新特征后数据量：{df.shape}')
# 衍生时间特征
if is_timefurture:
df = addtimecharacteristics(df=df, dataset=dataset)
if config.freq == 'WW':
# 自定义周数据
# 按weekofmothe分组取均值得到新的数据
df = df.groupby(df['yearmonthweeks']).mean()
# 时间列转换为日期格式字符串
df['ds'] = df['ds'].dt.strftime('%Y-%m-%d')
elif config.freq == 'W':
# 按周取样
df = df.resample('W', on='ds').mean().reset_index()
elif config.freq == 'M':
# 按月取样
if 'yearmonthweeks' in df.columns:
df.drop('yearmonthweeks', axis=1, inplace=True)
df = df.resample('ME', on='ds').mean().reset_index()
# 删除预测列空值的行
df = df.dropna(subset=['y'])
config.logger.info(f'删除预测列为空值的行后数据量：{df.shape}')
# df = df.dropna(subset=['y'])
# config.logger.info(f'删除预测列为空值的行后数据量：{df.shape}')
df = df.dropna(axis=1, how='all')
config.logger.info(f'删除全为空值的列后数据量：{df.shape}')
df.to_csv(os.path.join(dataset, '未填充的特征数据.csv'), index=False)
@ -1046,9 +1064,6 @@ def datachuli_juxiting(df_zhibiaoshuju, df_zhibiaoliebiao, datecol='date', end_t
if add_kdj:
df = calculate_kdj(df)
if is_timefurture:
df = addtimecharacteristics(df=df, dataset=dataset)
featureAnalysis(df, dataset=dataset, y=y)
return df
@ -1146,7 +1161,7 @@ class Config:
@property
def weight_dict(self): return global_config['weight_dict']
@property
def bdwdnames(self): return global_config['bdwdname']
def bdwdname(self): return global_config['bdwdname']
# 模型参数
@property
@ -1266,6 +1281,10 @@ class Config:
@property
def is_bdwd(self): return global_config['is_bdwd']
# 八大维度列名重命名
@property
def columnsrename(self): return global_config['columnsrename']
config = Config()
@ -1880,7 +1899,8 @@ class EtaReader():
'''
# 构建新的DataFrame df df1
df = pd.DataFrame(columns=['指标分类', '指标名称', '指标id', '频度'])
df = pd.DataFrame(columns=[
'指标分类', '指标名称', '指标id', '频度', '指标来源', '来源id', '最后更新时间', '更新周期', '预警日期', '停更周期'])
df1 = pd.DataFrame(columns=['DataTime'])
# 外网环境无法访问，请确认是否为内网环境
@ -1925,6 +1945,9 @@ class EtaReader():
# 指标名称，要保存到df2的指标名称列,df的指标名称列
EdbName = i.get('EdbName')
Frequency = i.get('Frequency') # 频度，要保存到df的频度列
SourceName = i.get('SourceName') # 来源名称，要保存到df的频度列
Source = i.get('Source') # 来源ID，要保存到df的频度列
Unit = i.get('Unit') # 单位，要保存到df的单位列
# 频度不是 日 或者 周的 跳过
if Frequency not in ['日度', '周度', '日', '周']:
continue
@ -1933,13 +1956,62 @@ class EtaReader():
isSave = self.filter_pp_data(ClassifyName, EdbName)
if isSave:
# 保存到df
df1 = self.edbcodegetdata(df1, EdbCode, EdbName)
# 取df1所有行最后一列
edbname_df = df1[['DataTime', f'{EdbName}']]
edbname_df = edbname_df.dropna()
if len(edbname_df) == 0:
config.logger.info(
f'指标名称：{EdbName} 没有数据')
continue
try:
time_sequence = edbname_df['DataTime'].values.tolist(
)[-10:]
except IndexError:
time_sequence = edbname_df['DataTime'].values.tolist(
)
# 使用Counter来统计每个星期几的出现次数
from collections import Counter
weekday_counter = Counter(datetime.datetime.strptime(
time_str, "%Y-%m-%d").strftime('%A') for time_str in time_sequence)
# 打印出现次数最多的星期几
try:
most_common_weekday = weekday_counter.most_common(1)[
0][0]
# 计算两周后的日期
warning_date = (datetime.datetime.strptime(
time_sequence[-1], "%Y-%m-%d") + datetime.timedelta(weeks=2)).strftime("%Y-%m-%d")
stop_update_period = (datetime.datetime.strptime(
today, "%Y-%m-%d") - datetime.datetime.strptime(time_sequence[-1], "%Y-%m-%d")).days // 7
except IndexError:
most_common_weekday = '其他'
stop_update_period = 0
if '日' in Frequency:
most_common_weekday = '每天'
warning_date = (datetime.datetime.strptime(
time_sequence[-1], "%Y-%m-%d") + datetime.timedelta(days=3)).strftime("%Y-%m-%d")
stop_update_period = (datetime.datetime.strptime(
today, "%Y-%m-%d") - datetime.datetime.strptime(time_sequence[-1], "%Y-%m-%d")).days
# 保存频度 指标名称 分类 指标id 到 df
df2 = pd.DataFrame(
{'指标分类': ClassifyName, '指标名称': EdbName, '指标id': EdbCode, '频度': Frequency}, index=[0])
df2 = pd.DataFrame({'指标分类': ClassifyName,
'指标名称': EdbName,
'指标id': EdbCode,
'单位': Unit,
'频度': Frequency,
'指标来源': SourceName,
'来源id': Source,
'最后更新时间': edbname_df['DataTime'].values[-1],
'更新周期': most_common_weekday,
'预警日期': warning_date,
'停更周期': stop_update_period}, index=[0],
)
# df = pd.merge(df, df2, how='outer')
df = pd.concat([df, df2])
df1 = self.edbcodegetdata(df1, EdbCode, EdbName)
else:
config.logger.info(f'跳过指标 {EdbName}')
@ -1958,7 +2030,7 @@ class EtaReader():
df1 = self.edbcodegetdata(df1, item, itemname)
df = pd.concat([df, pd.DataFrame(
{'指标分类': '其他', '指标名称': itemname, '指标id': item, '频度': '其他'}, index=[0])])
{'指标分类': '其他', '指标名称': itemname, '指标id': item, '频度': '其他', '指标来源': '其他', '来源id': '其他'}, index=[0])])
# 按时间排序
df1.sort_values('DataTime', inplace=True, ascending=False)
@ -1968,10 +2040,6 @@ class EtaReader():
df1 = df1[df1['date'] <= datetime.datetime.now().strftime('%Y-%m-%d')]
config.logger.info(df1.head())
# config.logger.info(f'{df1.head()}')
# 保存到xlsx文件的sheet表
with pd.ExcelWriter(os.path.join(dataset, data_set)) as file:
df1.to_excel(file, sheet_name='指标数据', index=False)
df.to_excel(file, sheet_name='指标列表', index=False)
df_zhibiaoshuju = df1.copy()
df_zhibiaoliebiao = df.copy()
@ -2217,8 +2285,9 @@ def push_market_data(data):
return json_data
def push_waring_market_data(data):
def push_waring_market_data(data,dataSource=8):
'''
dataSource : 8 原油 9 聚烯烃PP
上传停更预警数据到市场信息平台
data: 预测价格数据,示例：
{
@ -2244,6 +2313,7 @@ def push_waring_market_data(data):
token = get_head_auth_report()
# 定义请求参数
global_config['push_waring_data_value_list_data']['data']["crudeOilWarningDtoList"] = data
global_config['push_waring_data_value_list_data']['data']["dataSource"] = str(dataSource)
# 发送请求
headers = {"Authorization": token}
config.logger.info('上传数据中...')
@ -2453,8 +2523,7 @@ def get_bdwd_predict_data():
df2['date'] = df2['date'].dt.strftime('%Y-%m-%d')
# df = pd.merge(df, df2, how='left', on='date')
# 更改列名：
df2.rename(columns={'yyycbdwdbz': '本周', 'yyycbdwdcey': '次二月', 'yyycbdwdcr': '次日', 'yyycbdwdcsiy': '次四月',
'yyycbdwdcsy': '次三月', 'yyycbdwdcy': '次月', 'yyycbdwdcz': '次周', 'yyycbdwdgz': '隔周', }, inplace=True)
df2.rename(columns=config.columnsrename, inplace=True)
# df2.rename(columns={'原油大数据预测|FORECAST|PRICE|W':'本周','原油大数据预测|FORECAST|PRICE|M_2':'次二月','原油大数据预测|FORECAST|PRICE|T':'次日','原油大数据预测|FORECAST|PRICE|M_4':'次四月','原油大数据预测|FORECAST|PRICE|M_3':'次三月','原油大数据预测|FORECAST|PRICE|M_1':'次月','原油大数据预测|FORECAST|PRICE|W_1':'次周','原油大数据预测|FORECAST|PRICE|W_2':'隔周',}, inplace=True)
# 更改显示顺序
# 过滤掉不存在的列

80
main_juxiting.py
View File

@ -20,6 +20,7 @@ global_config.update({
'is_debug': is_debug,
'is_train': is_train,
'is_fivemodels': is_fivemodels,
'is_update_report': is_update_report,
'settings': settings,
@ -34,6 +35,7 @@ global_config.update({
'modelsindex': modelsindex,
'rote': rote,
'bdwd_items': bdwd_items,
'bdwdname':bdwdname,
# 特征工程开关
'is_del_corr': is_del_corr,
@ -52,6 +54,7 @@ global_config.update({
'login_pushreport_url': login_pushreport_url,
'login_data': login_data,
'upload_url': upload_url,
'upload_data': upload_data,
'upload_warning_url': upload_warning_url,
'warning_data': warning_data,
@ -63,6 +66,14 @@ global_config.update({
'push_data_value_list_url': push_data_value_list_url,
'push_data_value_list_data': push_data_value_list_data,
# 上传预警数据
'push_waring_data_value_list_url': push_waring_data_value_list_url,
'push_waring_data_value_list_data': push_waring_data_value_list_data,
# 获取取消订阅的数据
'get_waring_data_value_list_url': get_waring_data_value_list_url,
'get_waring_data_value_list_data': get_waring_data_value_list_data,
# eta 配置
'APPID': APPID,
'SECRET': SECRET,
@ -335,42 +346,41 @@ def predict_main():
sqlitedb.insert_data('most_model', (datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S'), most_common_model,), columns=('ds', 'most_common_model',))
try:
if is_weekday:
# if True:
logger.info('今天是周一，发送特征预警')
# 上传预警信息到数据库
warning_data_df = df_zhibiaoliebiao.copy()
warning_data_df = warning_data_df[warning_data_df['停更周期'] > 3][[
'指标名称', '指标id', '频度', '更新周期', '指标来源', '最后更新时间', '停更周期']]
# 重命名列名
warning_data_df = warning_data_df.rename(columns={'指标名称': 'INDICATOR_NAME', '指标id': 'INDICATOR_ID', '频度': 'FREQUENCY',
'更新周期': 'UPDATE_FREQUENCY', '指标来源': 'DATA_SOURCE', '最后更新时间': 'LAST_UPDATE_DATE', '停更周期': 'UPDATE_SUSPENSION_CYCLE'})
from sqlalchemy import create_engine
import urllib
global password
if '@' in password:
password = urllib.parse.quote_plus(password)
# try:
# if is_weekday:
if True:
logger.info('发送特征预警')
# 获取取消订阅的指标ID
quxiaodingyueidlist = get_waring_data()
# 上传预警信息到数据库
warning_data_df = df_zhibiaoliebiao.copy()
warning_data_df = warning_data_df[warning_data_df['停更周期'] > 3][[
'指标名称', '指标id', '频度', '更新周期', '指标来源', '最后更新时间', '停更周期']]
# 重命名列名
warning_data_df = warning_data_df.rename(columns={'指标名称': 'indicatorName', '指标id': 'indicatorId', '频度': 'frequency',
'更新周期': 'UPDATE_FREQUENCY', '指标来源': 'DATA_SOURCE', '最后更新时间': 'LAST_UPDATE_DATE', '停更周期': 'updateSuspensionCycle'})
engine = create_engine(
f'mysql+pymysql://{dbusername}:{password}@{host}:{port}/{dbname}')
warning_data_df['WARNING_DATE'] = datetime.date.today().strftime(
"%Y-%m-%d %H:%M:%S")
warning_data_df['TENANT_CODE'] = 'T0004'
# 插入数据之前查询表数据然后新增id列
existing_data = pd.read_sql(f"SELECT * FROM {table_name}", engine)
if not existing_data.empty:
max_id = existing_data['ID'].astype(int).max()
warning_data_df['ID'] = range(
max_id + 1, max_id + 1 + len(warning_data_df))
else:
warning_data_df['ID'] = range(1, 1 + len(warning_data_df))
warning_data_df.to_sql(
table_name, con=engine, if_exists='append', index=False)
if is_update_warning_data:
upload_warning_info(len(warning_data_df))
except:
logger.info('上传预警信息到数据库失败')
warning_data_df['warningDate'] = datetime.date.today().strftime(
"%Y-%m-%d %H:%M:%S")
warning_data_df['dataSource'] = 9
if len(quxiaodingyueidlist)>0:
# 去掉取消订阅的指标
print(warning_data_df.shape)
warning_data_df = warning_data_df[~warning_data_df['indicatorId'].isin(
quxiaodingyueidlist)]
print(warning_data_df.shape)
warning_data = warning_data_df.to_json(
orient='records', force_ascii=False)
warning_data = warning_data.replace('日度', '1')
warning_data = warning_data.replace('周度', '2')
warning_data = warning_data.replace('月度', '3')
warning_data = json.loads(warning_data)
push_waring_market_data(warning_data,dataSource=warning_data_df['dataSource'].values[0])
# if is_update_warning_data:
# upload_warning_info(len(warning_data_df))
# except:
# logger.info('上传预警信息到数据库失败')
if is_corr:
df = corr_feature(df=df)

469
main_juxiting_yuedu.py
View File

@ -20,7 +20,9 @@ global_config.update({
'is_debug': is_debug,
'is_train': is_train,
'is_fivemodels': is_fivemodels,
'is_update_report': is_update_report,
'settings': settings,
'bdwdname': bdwdname,
# 模型参数
@ -33,6 +35,7 @@ global_config.update({
'test_size': test_size,
'modelsindex': modelsindex,
'rote': rote,
'bdwd_items': bdwd_items,
# 特征工程开关
'is_del_corr': is_del_corr,
@ -40,6 +43,7 @@ global_config.update({
'is_eta': is_eta,
'is_update_eta': is_update_eta,
'is_fivemodels': is_fivemodels,
'is_update_predict_value': is_update_predict_value,
'early_stop_patience_steps': early_stop_patience_steps,
# 时间参数
@ -51,6 +55,7 @@ global_config.update({
'login_pushreport_url': login_pushreport_url,
'login_data': login_data,
'upload_url': upload_url,
'upload_data': upload_data,
'upload_warning_url': upload_warning_url,
'warning_data': warning_data,
@ -58,6 +63,10 @@ global_config.update({
'query_data_list_item_nos_url': query_data_list_item_nos_url,
'query_data_list_item_nos_data': query_data_list_item_nos_data,
# 上传数据项
'push_data_value_list_url': push_data_value_list_url,
'push_data_value_list_data': push_data_value_list_data,
# eta 配置
'APPID': APPID,
'SECRET': SECRET,
@ -74,9 +83,83 @@ global_config.update({
# 数据库配置
'sqlitedb': sqlitedb,
'is_bdwd': is_bdwd,
'columnsrename':columnsrename,
})
def push_market_value():
logger.info('发送预测结果到市场信息平台')
# 读取预测数据和模型评估数据
predict_file_path = os.path.join(config.dataset, 'predict.csv')
model_eval_file_path = os.path.join(config.dataset, 'model_evaluation.csv')
try:
predictdata_df = pd.read_csv(predict_file_path)
top_models_df = pd.read_csv(model_eval_file_path)
except FileNotFoundError as e:
logger.error(f"文件未找到: {e}")
return
predictdata = predictdata_df.copy()
# 取模型前十
top_models = top_models_df['模型(Model)'].head(10).tolist()
# 去掉FDBformer
if 'FEDformer' in top_models:
top_models.remove('FEDformer')
# 计算前十模型的均值
predictdata_df['top_models_mean'] = predictdata_df[top_models].mean(axis=1)
# 打印日期和前十模型均值
print(predictdata_df[['ds', 'top_models_mean']])
# 准备要推送的数据
ciyue_mean = predictdata_df['top_models_mean'].iloc[0]
cieryue_mean = predictdata_df['top_models_mean'].iloc[1]
cisanyue_mean = predictdata_df['top_models_mean'].iloc[2]
cisieryue_mean = predictdata_df['top_models_mean'].iloc[3]
# 保留两位小数
ciyue_mean = round(ciyue_mean, 2)
cieryue_mean = round(cieryue_mean, 2)
cisanyue_mean = round(cisanyue_mean, 2)
cisieryue_mean = round(cisieryue_mean, 2)
predictdata = [
{
"dataItemNo": global_config['bdwd_items']['ciyue'],
"dataDate": global_config['end_time'].replace('-', ''),
"dataStatus": "add",
"dataValue": ciyue_mean
},
{
"dataItemNo": global_config['bdwd_items']['cieryue'],
"dataDate": global_config['end_time'].replace('-', ''),
"dataStatus": "add",
"dataValue": cieryue_mean
},
{
"dataItemNo": global_config['bdwd_items']['cisanyue'],
"dataDate": global_config['end_time'].replace('-', ''),
"dataStatus": "add",
"dataValue": cisanyue_mean
},
{
"dataItemNo": global_config['bdwd_items']['cisiyue'],
"dataDate": global_config['end_time'].replace('-', ''),
"dataStatus": "add",
"dataValue": cisieryue_mean
}
]
print(predictdata)
# 推送数据到市场信息平台
try:
push_market_data(predictdata)
except Exception as e:
logger.error(f"推送数据失败: {e}")
def predict_main():
"""
主预测函数，用于从 ETA 获取数据、处理数据、训练模型并进行预测。
@ -116,234 +199,198 @@ def predict_main():
返回:
None
"""
end_time = global_config['end_time']
signature = BinanceAPI(APPID, SECRET)
etadata = EtaReader(signature=signature,
classifylisturl=global_config['classifylisturl'],
classifyidlisturl=global_config['classifyidlisturl'],
edbcodedataurl=global_config['edbcodedataurl'],
edbcodelist=global_config['edbcodelist'],
edbdatapushurl=global_config['edbdatapushurl'],
edbdeleteurl=global_config['edbdeleteurl'],
edbbusinessurl=global_config['edbbusinessurl'],
classifyId=global_config['ClassifyId'],
)
# 获取数据
if is_eta:
logger.info('从eta获取数据...')
# end_time = global_config['end_time']
# signature = BinanceAPI(APPID, SECRET)
# etadata = EtaReader(signature=signature,
# classifylisturl=global_config['classifylisturl'],
# classifyidlisturl=global_config['classifyidlisturl'],
# edbcodedataurl=global_config['edbcodedataurl'],
# edbcodelist=global_config['edbcodelist'],
# edbdatapushurl=global_config['edbdatapushurl'],
# edbdeleteurl=global_config['edbdeleteurl'],
# edbbusinessurl=global_config['edbbusinessurl'],
# classifyId=global_config['ClassifyId'],
# )
# # 获取数据
# if is_eta:
# logger.info('从eta获取数据...')
df_zhibiaoshuju, df_zhibiaoliebiao = etadata.get_eta_api_pp_data(
data_set=data_set, dataset=dataset) # 原始数据，未处理
# df_zhibiaoshuju, df_zhibiaoliebiao = etadata.get_eta_api_pp_data(
# data_set=data_set, dataset=dataset) # 原始数据，未处理
if is_market:
logger.info('从市场信息平台获取数据...')
try:
# 如果是测试环境，最高价最低价取excel文档
if server_host == '192.168.100.53':
logger.info('从excel文档获取最高价最低价')
df_zhibiaoshuju = get_high_low_data(df_zhibiaoshuju)
else:
logger.info('从市场信息平台获取数据')
df_zhibiaoshuju = get_market_data(
end_time, df_zhibiaoshuju)
# if is_market:
# logger.info('从市场信息平台获取数据...')
# try:
# # 如果是测试环境，最高价最低价取excel文档
# if server_host == '192.168.100.53':
# logger.info('从excel文档获取最高价最低价')
# df_zhibiaoshuju = get_high_low_data(df_zhibiaoshuju)
# else:
# logger.info('从市场信息平台获取数据')
# df_zhibiaoshuju = get_market_data(
# end_time, df_zhibiaoshuju)
except:
logger.info('最高最低价拼接失败')
# except:
# logger.info('最高最低价拼接失败')
# 保存到xlsx文件的sheet表
with pd.ExcelWriter(os.path.join(dataset, data_set)) as file:
df_zhibiaoshuju.to_excel(file, sheet_name='指标数据', index=False)
df_zhibiaoliebiao.to_excel(file, sheet_name='指标列表', index=False)
# # 保存到xlsx文件的sheet表
# with pd.ExcelWriter(os.path.join(dataset, data_set)) as file:
# df_zhibiaoshuju.to_excel(file, sheet_name='指标数据', index=False)
# df_zhibiaoliebiao.to_excel(file, sheet_name='指标列表', index=False)
# 数据处理
df = datachuli_juxiting(df_zhibiaoshuju, df_zhibiaoliebiao, y=global_config['y'], dataset=dataset, add_kdj=add_kdj, is_timefurture=is_timefurture,
end_time=end_time)
# # 数据处理
# df = datachuli_juxiting(df_zhibiaoshuju, df_zhibiaoliebiao, y=global_config['y'], dataset=dataset, add_kdj=add_kdj, is_timefurture=is_timefurture,
# end_time=end_time)
else:
# 读取数据
logger.info('读取本地数据：' + os.path.join(dataset, data_set))
df, df_zhibiaoliebiao = getdata_zhoudu_juxiting(filename=os.path.join(dataset, data_set), y=y, dataset=dataset, add_kdj=add_kdj,
is_timefurture=is_timefurture, end_time=end_time) # 原始数据，未处理
# else:
# # 读取数据
# logger.info('读取本地数据：' + os.path.join(dataset, data_set))
# df, df_zhibiaoliebiao = getdata_zhoudu_juxiting(filename=os.path.join(dataset, data_set), y=y, dataset=dataset, add_kdj=add_kdj,
# is_timefurture=is_timefurture, end_time=end_time) # 原始数据，未处理
# 更改预测列名称
df.rename(columns={y: 'y'}, inplace=True)
# # 更改预测列名称
# df.rename(columns={y: 'y'}, inplace=True)
if is_edbnamelist:
df = df[edbnamelist]
df.to_csv(os.path.join(dataset, '指标数据.csv'), index=False)
# 保存最新日期的y值到数据库
# 取第一行数据存储到数据库中
first_row = df[['ds', 'y']].tail(1)
# 判断y的类型是否为float
if not isinstance(first_row['y'].values[0], float):
logger.info(f'{end_time}预测目标数据为空，跳过')
return None
# if is_edbnamelist:
# df = df[edbnamelist]
# df.to_csv(os.path.join(dataset, '指标数据.csv'), index=False)
# # 保存最新日期的y值到数据库
# # 取第一行数据存储到数据库中
# first_row = df[['ds', 'y']].tail(1)
# # 判断y的类型是否为float
# if not isinstance(first_row['y'].values[0], float):
# logger.info(f'{end_time}预测目标数据为空，跳过')
# return None
# 将最新真实值保存到数据库
if not sqlitedb.check_table_exists('trueandpredict'):
first_row.to_sql('trueandpredict', sqlitedb.connection, index=False)
else:
for row in first_row.itertuples(index=False):
row_dict = row._asdict()
config.logger.info(f'要保存的真实值：{row_dict}')
# 判断ds是否为字符串类型,如果不是则转换为字符串类型
if isinstance(row_dict['ds'], (pd.Timestamp, datetime.datetime)):
row_dict['ds'] = row_dict['ds'].strftime('%Y-%m-%d')
elif not isinstance(row_dict['ds'], str):
try:
row_dict['ds'] = pd.to_datetime(
row_dict['ds']).strftime('%Y-%m-%d')
except:
logger.warning(f"无法解析的时间格式: {row_dict['ds']}")
# row_dict['ds'] = row_dict['ds'].strftime('%Y-%m-%d')
# row_dict['ds'] = row_dict['ds'].strftime('%Y-%m-%d %H:%M:%S')
check_query = sqlitedb.select_data(
'trueandpredict', where_condition=f"ds = '{row.ds}'")
if len(check_query) > 0:
set_clause = ", ".join(
[f"{key} = '{value}'" for key, value in row_dict.items()])
sqlitedb.update_data(
'trueandpredict', set_clause, where_condition=f"ds = '{row.ds}'")
continue
sqlitedb.insert_data('trueandpredict', tuple(
row_dict.values()), columns=row_dict.keys())
# # 将最新真实值保存到数据库
# if not sqlitedb.check_table_exists('trueandpredict'):
# first_row.to_sql('trueandpredict', sqlitedb.connection, index=False)
# else:
# for row in first_row.itertuples(index=False):
# row_dict = row._asdict()
# config.logger.info(f'要保存的真实值：{row_dict}')
# # 判断ds是否为字符串类型,如果不是则转换为字符串类型
# if isinstance(row_dict['ds'], (pd.Timestamp, datetime.datetime)):
# row_dict['ds'] = row_dict['ds'].strftime('%Y-%m-%d')
# elif not isinstance(row_dict['ds'], str):
# try:
# row_dict['ds'] = pd.to_datetime(
# row_dict['ds']).strftime('%Y-%m-%d')
# except:
# logger.warning(f"无法解析的时间格式: {row_dict['ds']}")
# # row_dict['ds'] = row_dict['ds'].strftime('%Y-%m-%d')
# # row_dict['ds'] = row_dict['ds'].strftime('%Y-%m-%d %H:%M:%S')
# check_query = sqlitedb.select_data(
# 'trueandpredict', where_condition=f"ds = '{row.ds}'")
# if len(check_query) > 0:
# set_clause = ", ".join(
# [f"{key} = '{value}'" for key, value in row_dict.items()])
# sqlitedb.update_data(
# 'trueandpredict', set_clause, where_condition=f"ds = '{row.ds}'")
# continue
# sqlitedb.insert_data('trueandpredict', tuple(
# row_dict.values()), columns=row_dict.keys())
# 更新accuracy表的y值
if not sqlitedb.check_table_exists('accuracy'):
pass
else:
update_y = sqlitedb.select_data(
'accuracy', where_condition="y is null")
if len(update_y) > 0:
logger.info('更新accuracy表的y值')
# 找到update_y 中ds且df中的y的行
update_y = update_y[update_y['ds'] <= end_time]
logger.info(f'要更新y的信息：{update_y}')
# try:
for row in update_y.itertuples(index=False):
try:
row_dict = row._asdict()
yy = df[df['ds'] == row_dict['ds']]['y'].values[0]
LOW = df[df['ds'] == row_dict['ds']]['Brentzdj'].values[0]
HIGH = df[df['ds'] == row_dict['ds']]['Brentzgj'].values[0]
sqlitedb.update_data(
'accuracy', f"y = {yy},LOW_PRICE = {LOW},HIGH_PRICE = {HIGH}", where_condition=f"ds = '{row_dict['ds']}'")
except:
logger.info(f'更新accuracy表的y值失败：{row_dict}')
# except Exception as e:
# logger.info(f'更新accuracy表的y值失败：{e}')
# # 更新accuracy表的y值
# if not sqlitedb.check_table_exists('accuracy'):
# pass
# else:
# update_y = sqlitedb.select_data(
# 'accuracy', where_condition="y is null")
# if len(update_y) > 0:
# logger.info('更新accuracy表的y值')
# # 找到update_y 中ds且df中的y的行
# update_y = update_y[update_y['ds'] <= end_time]
# logger.info(f'要更新y的信息：{update_y}')
# # try:
# for row in update_y.itertuples(index=False):
# try:
# row_dict = row._asdict()
# yy = df[df['ds'] == row_dict['ds']]['y'].values[0]
# LOW = df[df['ds'] == row_dict['ds']]['Brentzdj'].values[0]
# HIGH = df[df['ds'] == row_dict['ds']]['Brentzgj'].values[0]
# sqlitedb.update_data(
# 'accuracy', f"y = {yy},LOW_PRICE = {LOW},HIGH_PRICE = {HIGH}", where_condition=f"ds = '{row_dict['ds']}'")
# except:
# logger.info(f'更新accuracy表的y值失败：{row_dict}')
# # except Exception as e:
# # logger.info(f'更新accuracy表的y值失败：{e}')
# 判断当前日期是不是周一
is_weekday = datetime.datetime.now().weekday() == 0
if is_weekday:
logger.info('今天是周一，更新预测模型')
# 计算最近60天预测残差最低的模型名称
model_results = sqlitedb.select_data(
'trueandpredict', order_by="ds DESC", limit="60")
# 删除空值率为90%以上的列
if len(model_results) > 10:
model_results = model_results.dropna(
thresh=len(model_results)*0.1, axis=1)
# 删除空行
model_results = model_results.dropna()
modelnames = model_results.columns.to_list()[2:-1]
for col in model_results[modelnames].select_dtypes(include=['object']).columns:
model_results[col] = model_results[col].astype(np.float32)
# 计算每个预测值与真实值之间的偏差率
for model in modelnames:
model_results[f'{model}_abs_error_rate'] = abs(
model_results['y'] - model_results[model]) / model_results['y']
# 获取每行对应的最小偏差率值
min_abs_error_rate_values = model_results.apply(
lambda row: row[[f'{model}_abs_error_rate' for model in modelnames]].min(), axis=1)
# 获取每行对应的最小偏差率值对应的列名
min_abs_error_rate_column_name = model_results.apply(
lambda row: row[[f'{model}_abs_error_rate' for model in modelnames]].idxmin(), axis=1)
# 将列名索引转换为列名
min_abs_error_rate_column_name = min_abs_error_rate_column_name.map(
lambda x: x.split('_')[0])
# 取出现次数最多的模型名称
most_common_model = min_abs_error_rate_column_name.value_counts().idxmax()
logger.info(f"最近60天预测残差最低的模型名称：{most_common_model}")
# 保存结果到数据库
if not sqlitedb.check_table_exists('most_model'):
sqlitedb.create_table(
'most_model', columns="ds datetime, most_common_model TEXT")
sqlitedb.insert_data('most_model', (datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S'), most_common_model,), columns=('ds', 'most_common_model',))
# # 判断当前日期是不是周一
# is_weekday = datetime.datetime.now().weekday() == 0
# if is_weekday:
# logger.info('今天是周一，更新预测模型')
# # 计算最近60天预测残差最低的模型名称
# model_results = sqlitedb.select_data(
# 'trueandpredict', order_by="ds DESC", limit="60")
# # 删除空值率为90%以上的列
# if len(model_results) > 10:
# model_results = model_results.dropna(
# thresh=len(model_results)*0.1, axis=1)
# # 删除空行
# model_results = model_results.dropna()
# modelnames = model_results.columns.to_list()[2:-1]
# for col in model_results[modelnames].select_dtypes(include=['object']).columns:
# model_results[col] = model_results[col].astype(np.float32)
# # 计算每个预测值与真实值之间的偏差率
# for model in modelnames:
# model_results[f'{model}_abs_error_rate'] = abs(
# model_results['y'] - model_results[model]) / model_results['y']
# # 获取每行对应的最小偏差率值
# min_abs_error_rate_values = model_results.apply(
# lambda row: row[[f'{model}_abs_error_rate' for model in modelnames]].min(), axis=1)
# # 获取每行对应的最小偏差率值对应的列名
# min_abs_error_rate_column_name = model_results.apply(
# lambda row: row[[f'{model}_abs_error_rate' for model in modelnames]].idxmin(), axis=1)
# # 将列名索引转换为列名
# min_abs_error_rate_column_name = min_abs_error_rate_column_name.map(
# lambda x: x.split('_')[0])
# # 取出现次数最多的模型名称
# most_common_model = min_abs_error_rate_column_name.value_counts().idxmax()
# logger.info(f"最近60天预测残差最低的模型名称：{most_common_model}")
# # 保存结果到数据库
# if not sqlitedb.check_table_exists('most_model'):
# sqlitedb.create_table(
# 'most_model', columns="ds datetime, most_common_model TEXT")
# sqlitedb.insert_data('most_model', (datetime.datetime.now().strftime(
# '%Y-%m-%d %H:%M:%S'), most_common_model,), columns=('ds', 'most_common_model',))
try:
if is_weekday:
# if True:
logger.info('今天是周一，发送特征预警')
# 上传预警信息到数据库
warning_data_df = df_zhibiaoliebiao.copy()
warning_data_df = warning_data_df[warning_data_df['停更周期'] > 3][[
'指标名称', '指标id', '频度', '更新周期', '指标来源', '最后更新时间', '停更周期']]
# 重命名列名
warning_data_df = warning_data_df.rename(columns={'指标名称': 'INDICATOR_NAME', '指标id': 'INDICATOR_ID', '频度': 'FREQUENCY',
'更新周期': 'UPDATE_FREQUENCY', '指标来源': 'DATA_SOURCE', '最后更新时间': 'LAST_UPDATE_DATE', '停更周期': 'UPDATE_SUSPENSION_CYCLE'})
from sqlalchemy import create_engine
import urllib
global password
if '@' in password:
password = urllib.parse.quote_plus(password)
# if is_corr:
# df = corr_feature(df=df)
engine = create_engine(
f'mysql+pymysql://{dbusername}:{password}@{host}:{port}/{dbname}')
warning_data_df['WARNING_DATE'] = datetime.date.today().strftime(
"%Y-%m-%d %H:%M:%S")
warning_data_df['TENANT_CODE'] = 'T0004'
# 插入数据之前查询表数据然后新增id列
existing_data = pd.read_sql(f"SELECT * FROM {table_name}", engine)
if not existing_data.empty:
max_id = existing_data['ID'].astype(int).max()
warning_data_df['ID'] = range(
max_id + 1, max_id + 1 + len(warning_data_df))
else:
warning_data_df['ID'] = range(1, 1 + len(warning_data_df))
warning_data_df.to_sql(
table_name, con=engine, if_exists='append', index=False)
if is_update_warning_data:
upload_warning_info(len(warning_data_df))
except:
logger.info('上传预警信息到数据库失败')
# df1 = df.copy() # 备份一下，后面特征筛选完之后加入ds y 列用
# logger.info(f"开始训练模型...")
# row, col = df.shape
if is_corr:
df = corr_feature(df=df)
# now = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
# ex_Model(df,
# horizon=global_config['horizon'],
# input_size=global_config['input_size'],
# train_steps=global_config['train_steps'],
# val_check_steps=global_config['val_check_steps'],
# early_stop_patience_steps=global_config['early_stop_patience_steps'],
# is_debug=global_config['is_debug'],
# dataset=global_config['dataset'],
# is_train=global_config['is_train'],
# is_fivemodels=global_config['is_fivemodels'],
# val_size=global_config['val_size'],
# test_size=global_config['test_size'],
# settings=global_config['settings'],
# now=now,
# etadata=etadata,
# modelsindex=global_config['modelsindex'],
# data=data,
# is_eta=global_config['is_eta'],
# end_time=global_config['end_time'],
# )
df1 = df.copy() # 备份一下，后面特征筛选完之后加入ds y 列用
logger.info(f"开始训练模型...")
row, col = df.shape
# logger.info('模型训练完成')
now = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
ex_Model(df,
horizon=global_config['horizon'],
input_size=global_config['input_size'],
train_steps=global_config['train_steps'],
val_check_steps=global_config['val_check_steps'],
early_stop_patience_steps=global_config['early_stop_patience_steps'],
is_debug=global_config['is_debug'],
dataset=global_config['dataset'],
is_train=global_config['is_train'],
is_fivemodels=global_config['is_fivemodels'],
val_size=global_config['val_size'],
test_size=global_config['test_size'],
settings=global_config['settings'],
now=now,
etadata=etadata,
modelsindex=global_config['modelsindex'],
data=data,
is_eta=global_config['is_eta'],
end_time=global_config['end_time'],
)
logger.info('模型训练完成')
logger.info('训练数据绘图ing')
model_results3 = model_losss_juxiting(sqlitedb, end_time=global_config['end_time'],is_fivemodels=global_config['is_fivemodels'])
logger.info('训练数据绘图end')
# logger.info('训练数据绘图ing')
# model_results3 = model_losss_juxiting(sqlitedb, end_time=global_config['end_time'],is_fivemodels=global_config['is_fivemodels'])
# logger.info('训练数据绘图end')
# push_market_value()
# # 模型报告
logger.info('制作报告ing')
title = f'{settings}--{end_time}-预测报告' # 报告标题

112
main_juxiting_zhoudu.py
View File

@ -20,7 +20,9 @@ global_config.update({
'is_debug': is_debug,
'is_train': is_train,
'is_fivemodels': is_fivemodels,
'is_update_report': is_update_report,
'settings': settings,
'bdwdname': bdwdname,
# 模型参数
@ -33,6 +35,7 @@ global_config.update({
'test_size': test_size,
'modelsindex': modelsindex,
'rote': rote,
'bdwd_items': bdwd_items,
# 特征工程开关
'is_del_corr': is_del_corr,
@ -40,6 +43,7 @@ global_config.update({
'is_eta': is_eta,
'is_update_eta': is_update_eta,
'is_fivemodels': is_fivemodels,
'is_update_predict_value': is_update_predict_value,
'early_stop_patience_steps': early_stop_patience_steps,
# 时间参数
@ -51,6 +55,7 @@ global_config.update({
'login_pushreport_url': login_pushreport_url,
'login_data': login_data,
'upload_url': upload_url,
'upload_data': upload_data,
'upload_warning_url': upload_warning_url,
'warning_data': warning_data,
@ -58,6 +63,10 @@ global_config.update({
'query_data_list_item_nos_url': query_data_list_item_nos_url,
'query_data_list_item_nos_data': query_data_list_item_nos_data,
# 上传数据项
'push_data_value_list_url': push_data_value_list_url,
'push_data_value_list_data': push_data_value_list_data,
# eta 配置
'APPID': APPID,
'SECRET': SECRET,
@ -74,9 +83,66 @@ global_config.update({
# 数据库配置
'sqlitedb': sqlitedb,
'is_bdwd': is_bdwd,
})
def push_market_value():
logger.info('发送预测结果到市场信息平台')
# 读取预测数据和模型评估数据
predict_file_path = os.path.join(config.dataset, 'predict.csv')
model_eval_file_path = os.path.join(config.dataset, 'model_evaluation.csv')
try:
predictdata_df = pd.read_csv(predict_file_path)
top_models_df = pd.read_csv(model_eval_file_path)
except FileNotFoundError as e:
logger.error(f"文件未找到: {e}")
return
predictdata = predictdata_df.copy()
# 取模型前十
top_models = top_models_df['模型(Model)'].head(10).tolist()
# 去掉FDBformer
if 'FEDformer' in top_models:
top_models.remove('FEDformer')
# 计算前十模型的均值
predictdata_df['top_models_mean'] = predictdata_df[top_models].mean(axis=1)
# 打印日期和前十模型均值
print(predictdata_df[['ds', 'top_models_mean']])
# 准备要推送的数据
first_mean = predictdata_df['top_models_mean'].iloc[0]
last_mean = predictdata_df['top_models_mean'].iloc[-1]
# 保留两位小数
first_mean = round(first_mean, 2)
last_mean = round(last_mean, 2)
predictdata = [
{
"dataItemNo": global_config['bdwd_items']['cizhou'],
"dataDate": global_config['end_time'].replace('-', ''),
"dataStatus": "add",
"dataValue": first_mean
},
{
"dataItemNo": global_config['bdwd_items']['gezhou'],
"dataDate": global_config['end_time'].replace('-', ''),
"dataStatus": "add",
"dataValue": last_mean
}
]
print(predictdata)
# 推送数据到市场信息平台
try:
push_market_data(predictdata)
except Exception as e:
logger.error(f"推送数据失败: {e}")
def predict_main():
"""
主预测函数，用于从 ETA 获取数据、处理数据、训练模型并进行预测。
@ -163,7 +229,7 @@ def predict_main():
# 读取数据
logger.info('读取本地数据：' + os.path.join(dataset, data_set))
df, df_zhibiaoliebiao = getdata_zhoudu_juxiting(filename=os.path.join(dataset, data_set), y=y, dataset=dataset, add_kdj=add_kdj,
is_timefurture=is_timefurture, end_time=end_time) # 原始数据，未处理
is_timefurture=is_timefurture, end_time=end_time) # 原始数据，未处理
# 更改预测列名称
df.rename(columns={y: 'y'}, inplace=True)
@ -246,7 +312,7 @@ def predict_main():
thresh=len(model_results)*0.1, axis=1)
# 删除空行
model_results = model_results.dropna()
modelnames = model_results.columns.to_list()[2:-1]
modelnames = model_results.columns.to_list()[2:-2]
for col in model_results[modelnames].select_dtypes(include=['object']).columns:
model_results[col] = model_results[col].astype(np.float32)
# 计算每个预测值与真实值之间的偏差率
@ -272,43 +338,6 @@ def predict_main():
sqlitedb.insert_data('most_model', (datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S'), most_common_model,), columns=('ds', 'most_common_model',))
try:
if is_weekday:
# if True:
logger.info('今天是周一，发送特征预警')
# 上传预警信息到数据库
warning_data_df = df_zhibiaoliebiao.copy()
warning_data_df = warning_data_df[warning_data_df['停更周期'] > 3][[
'指标名称', '指标id', '频度', '更新周期', '指标来源', '最后更新时间', '停更周期']]
# 重命名列名
warning_data_df = warning_data_df.rename(columns={'指标名称': 'INDICATOR_NAME', '指标id': 'INDICATOR_ID', '频度': 'FREQUENCY',
'更新周期': 'UPDATE_FREQUENCY', '指标来源': 'DATA_SOURCE', '最后更新时间': 'LAST_UPDATE_DATE', '停更周期': 'UPDATE_SUSPENSION_CYCLE'})
from sqlalchemy import create_engine
import urllib
global password
if '@' in password:
password = urllib.parse.quote_plus(password)
engine = create_engine(
f'mysql+pymysql://{dbusername}:{password}@{host}:{port}/{dbname}')
warning_data_df['WARNING_DATE'] = datetime.date.today().strftime(
"%Y-%m-%d %H:%M:%S")
warning_data_df['TENANT_CODE'] = 'T0004'
# 插入数据之前查询表数据然后新增id列
existing_data = pd.read_sql(f"SELECT * FROM {table_name}", engine)
if not existing_data.empty:
max_id = existing_data['ID'].astype(int).max()
warning_data_df['ID'] = range(
max_id + 1, max_id + 1 + len(warning_data_df))
else:
warning_data_df['ID'] = range(1, 1 + len(warning_data_df))
warning_data_df.to_sql(
table_name, con=engine, if_exists='append', index=False)
if is_update_warning_data:
upload_warning_info(len(warning_data_df))
except:
logger.info('上传预警信息到数据库失败')
if is_corr:
df = corr_feature(df=df)
@ -341,7 +370,8 @@ def predict_main():
logger.info('模型训练完成')
logger.info('训练数据绘图ing')
model_results3 = model_losss_juxiting(sqlitedb, end_time=global_config['end_time'],is_fivemodels=global_config['is_fivemodels'])
model_results3 = model_losss_juxiting(
sqlitedb, end_time=global_config['end_time'], is_fivemodels=global_config['is_fivemodels'])
logger.info('训练数据绘图end')
# # 模型报告
@ -355,6 +385,8 @@ def predict_main():
logger.info('制作报告end')
logger.info('模型训练完成')
push_market_value()
# # LSTM 单变量模型
# ex_Lstm(df,input_seq_len=input_size,output_seq_len=horizon,is_debug=is_debug,dataset=dataset)

86
models/nerulforcastmodels.py
View File

@ -245,7 +245,7 @@ def ex_Model(df, horizon, input_size, train_steps, val_check_steps, early_stop_p
IndexName = data['IndexName']
for index, row in df_predict.iterrows():
try:
yy = config.bdwdnames[index]
yy = config.bdwdname[index]
except IndexError as e:
break
for m in modelsindex[index].keys():
@ -471,26 +471,34 @@ def ex_Model_Juxiting(df, horizon, input_size, train_steps, val_check_steps, ear
# 把预测值上传到eta
if config.is_update_eta:
df_predict['ds'] = pd.to_datetime(df_predict['ds'])
dates = df_predict['ds'].dt.strftime('%Y-%m-%d')
for m in modelsindex.keys():
# 按行遍历df_predict
IndexName = data['IndexName']
for index, row in df_predict.iterrows():
try:
list = []
for date, value in zip(dates, df_predict[m].round(2)):
list.append({'Date': date, 'Value': value})
data['DataList'] = [list[-1]]
data['IndexCode'] = modelsindex[m]
data['IndexName'] = f'聚烯烃价格预测{m}模型'
yy = config.bdwdname[index]
except IndexError as e:
break
for m in modelsindex[index].keys():
if m not in df_predict.columns:
continue
if m == 'FEDformer':
continue
list = [{'Date': config.end_time, 'Value': round(row[m], 2)}]
data['DataList'] = list
data['IndexCode'] = modelsindex[index][m]
# data['IndexName'] = f'价格预测{m}模型'
data['IndexName'] = data['IndexName'].replace('xx', m)
data['IndexName'] = data['IndexName'].replace('yy', yy)
data['Remark'] = m
etadata.push_data(data=data)
except Exception as e:
config.logger.error(f'上传eta数据失败：{e}')
# return nf_test_preds
print('预测数据上传到eta:')
etadata.push_data(data)
# print(data)
data['IndexName'] = IndexName
return
# 雍安环境预测评估指数
@exception_logger
def model_losss_yongan(sqlitedb, end_time, table_name_prefix):
@ -2018,9 +2026,13 @@ def model_losss_juxiting(sqlitedb, end_time, is_fivemodels):
names = []
names_df = df_combined3.copy()
for col in allmodelnames:
names_df[f'{col}-{most_model_name}-误差比例'] = abs(
names_df[col] - names_df[most_model_name]) / names_df[most_model_name]
names.append(f'{col}-{most_model_name}-误差比例')
try:
names_df[f'{col}-{most_model_name}-误差比例'] = abs(
names_df[col] - names_df[most_model_name]) / names_df[most_model_name]
names.append(f'{col}-{most_model_name}-误差比例')
except KeyError:
print(f'{col}-{most_model_name}-不存在')
pass
names_df = names_df[names]
@ -3221,6 +3233,8 @@ def pp_export_pdf(num_indicators=475, num_models=21, num_dayindicator=202, input
col = col.replace('*', '-')
col = col.replace(':', '-')
col = col.replace(r'/', '-')
col = col.replace(r'>', '-')
col = col.replace(r'<', '-')
plt.savefig(os.path.join(config.dataset, f'{col}与价格散点图.png'))
content.append(Graphs.draw_img(
os.path.join(config.dataset, f'{col}与价格散点图.png')))
@ -3247,8 +3261,12 @@ def pp_export_pdf(num_indicators=475, num_models=21, num_dayindicator=202, input
config.dataset, 'model_evaluation.csv'), encoding='utf-8')
# 按评估指标排序，取前五
fivemodels_list = eval_df['模型(Model)'].values # 列表形式，后面当作列名索引使用
eval_models_list = fivemodels_list.tolist()
# 去掉FEDformer
if 'FEDformer' in eval_models_list:
eval_models_list.remove('FEDformer')
# 取 fivemodels_list 和 ds 列
df = df[['ds'] + fivemodels_list.tolist()]
df = df[['ds'] + eval_models_list]
# 拼接预测日期对应的真实值
df = pd.merge(df, df_true, on='ds', how='left')
# 删除全部为nan的列
@ -3270,18 +3288,32 @@ def pp_export_pdf(num_indicators=475, num_models=21, num_dayindicator=202, input
data = df.values.tolist()
col_width = 500/len(df.columns)
content.append(Graphs.draw_table(col_width, *data))
if config.is_bdwd:
# 添加八大维度数据
content.append(Graphs.draw_text('八大维度预测：'))
df_bdwd = get_bdwd_predict_data()
# df转置
df_bdwd = df_bdwd.T
# df重置索引
df_bdwd = df_bdwd.reset_index()
# 添加预测值表格
data = df_bdwd.values.tolist()
col_width = 500/len(df.columns)
content.append(Graphs.draw_table(col_width, *data))
pass
content.append(Graphs.draw_little_title('二、上一预测周期偏差率分析：'))
df = pd.read_csv(os.path.join(
config.dataset, 'testandpredict_groupby.csv'), encoding='utf-8')
df4 = df.copy() # 计算偏差率使用
# 去掉created_dt 列
df4 = df4.drop(columns=['created_dt'])
# 计算模型偏差率
# 计算各列对于y列的差值百分比
df3 = pd.DataFrame() # 存储偏差率
# 删除y列有空值的行
df4 = df4.dropna(subset=['y'])
# # 删除有null的行
# df4 = df4.dropna()
# 删除有null的行
df4 = df4.dropna()
df3['ds'] = df4['ds']
for col in fivemodels_list:
df3[col] = round(abs(df4[col] - df4['y']) / df4['y'] * 100, 2)
@ -3305,9 +3337,9 @@ def pp_export_pdf(num_indicators=475, num_models=21, num_dayindicator=202, input
# 特征、模型、参数配置
content.append(Graphs.draw_little_title('模型选择：'))
content.append(Graphs.draw_text(
f'本次预测调用专用于时间序列预测的NeuralForecast库中{num_models}个模型：'))
f'本次预测使用了一个专门收集时间序列的NeuralForecast库中的{num_models}个模型：'))
content.append(Graphs.draw_text(
f'基于40天历史数据构建多维时间窗口，采用注意力机制预测未来{inputsize}天趋势'))
f'使用{config.input_size}个数据预测未来{inputsize}个数据。'))
content.append(Graphs.draw_little_title('指标情况：'))
with open(os.path.join(config.dataset, '特征频度统计.txt'), encoding='utf-8') as f:
for line in f.readlines():
@ -3475,8 +3507,8 @@ def pp_export_pdf(num_indicators=475, num_models=21, num_dayindicator=202, input
if config.is_update_report:
with open(os.path.join(config.dataset, reportname), 'rb') as f:
base64_data = base64.b64encode(f.read()).decode('utf-8')
global_config['upload_data']["data"]["fileBase64"] = base64_data
global_config['upload_data']["data"]["fileName"] = reportname
config.upload_data["data"]["fileBase64"] = base64_data
config.upload_data["data"]["fileName"] = reportname
token = get_head_auth_report()
upload_report_data(token, config.upload_data)
except TimeoutError as e:

BIN
requirements.txt
View File

Binary file not shown.

2
原油八大维度预测任务.py
View File

@ -9,7 +9,7 @@ import time
def run_predictions(target_date):
"""执行三个预测脚本"""
scripts = [
# "main_yuanyou.py",
"main_yuanyou.py",
"main_yuanyou_zhoudu.py",
"main_yuanyou_yuedu.py"
]

3
聚烯烃八大维度预测任务.py
View File

@ -16,7 +16,8 @@ def run_predictions(target_date):
# 依次执行每个脚本
for script in scripts:
command = [r"C:\Users\Hello\.conda\envs\predict\python", script]
# command = [r"C:\Users\Hello\.conda\envs\predict\python", script] # liuruipc
command = [r"C:/Users/EDY/.conda/envs/priceforecast/python.exe", script] #168pc
subprocess.run(command, check=True)