添加原油特征停更预警

config_jingbo.py

@@ -121,38 +121,49 @@ upload_data = {
 
 
 # # 变量定义--测试环境
-# login_pushreport_url = "http://192.168.100.53:8080/jingbo-dev/api/server/login"
+login_pushreport_url = "http://192.168.100.53:8080/jingbo-dev/api/server/login"
-# upload_url = "http://192.168.100.53:8080/jingbo-dev/api/analysis/reportInfo/researchUploadReportSave"
+upload_url = "http://192.168.100.53:8080/jingbo-dev/api/analysis/reportInfo/researchUploadReportSave"
-# # upload_url = "http://192.168.100.109:8080/jingbo/api/analysis/reportInfo/researchUploadReportSave" # zhaoqiwei
+# upload_url = "http://192.168.100.109:8080/jingbo/api/analysis/reportInfo/researchUploadReportSave" # zhaoqiwei
+upload_warning_url = "http://192.168.100.53:8080/jingbo-dev/api/basicBuiness/crudeOilWarning/save"
 
 
-# login_data = {
+login_data = {
-#     "data": {
+    "data": {
-#         "account": "api_test",
+        "account": "api_test",
-#         "password": "MmVmNzNlOWI0MmY0ZDdjZGUwNzE3ZjFiMDJiZDZjZWU=",
+        # "password": "MmVmNzNlOWI0MmY0ZDdjZGUwNzE3ZjFiMDJiZDZjZWU=", # Shihua@123456
-#         "tenantHashCode": "8a4577dbd919675758d57999a1e891fe",
+        "password": "ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=", # 123456
-#         "terminal": "API"
+        "tenantHashCode": "8a4577dbd919675758d57999a1e891fe",
-#     },
+        "terminal": "API"
-#     "funcModule": "API",
+    },
-#     "funcOperation": "获取token"
+    "funcModule": "API",
-# }
+    "funcOperation": "获取token"
+}
 
-# upload_data = {
+upload_data = {
-#     "funcModule":'研究报告信息',
+    "funcModule":'研究报告信息',
-#     "funcOperation":'上传原油价格预测报告',
+    "funcOperation":'上传原油价格预测报告',
-#     "data":{
+    "data":{
-#         "ownerAccount":'arui', #报告所属用户账号
+        "ownerAccount":'arui', #报告所属用户账号
-#         "reportType":'OIL_PRICE_FORECAST', # 报告类型，固定为OIL_PRICE_FORECAST
+        "reportType":'OIL_PRICE_FORECAST', # 报告类型，固定为OIL_PRICE_FORECAST
-#         "fileName": '2000-40-5-50--100-原油指标数据.xlsx-Brent活跃合约--2024-09-06-15-01-29-预测报告.pdf', #文件名称 
+        "fileName": '2000-40-5-50--100-原油指标数据.xlsx-Brent活跃合约--2024-09-06-15-01-29-预测报告.pdf', #文件名称 
-#         "fileBase64": '' ,#文件内容base64
+        "fileBase64": '' ,#文件内容base64
-#         "categoryNo":'yyjgycbg', # 研究报告分类编码
+        "categoryNo":'yyjgycbg', # 研究报告分类编码
-#         "smartBusinessClassCode":'YCJGYCBG', #分析报告分类编码
+        "smartBusinessClassCode":'YCJGYCBG', #分析报告分类编码
-#         "reportEmployeeCode":"E40116", # 报告人
+        "reportEmployeeCode":"E40116", # 报告人
-#         "reportDeptCode" :"D0044" ,# 报告部门
+        "reportDeptCode" :"D0044" ,# 报告部门
-#         "productGroupCode":"RAW_MATERIAL"  # 商品分类
+        "productGroupCode":"RAW_MATERIAL"  # 商品分类
-#   }
+  }
-# }
+}
 
+warning_data = {
+    "funcModule":'原油特征停更预警',
+    "funcOperation":'原油特征停更预警',
+    "data":{
+    'WARNING_TYPE_NAME':'日度数据',
+    'WARNING_CONTENT':'',
+    'WARNING_DATE':''
+  }
+}
 
 ### 线上开关
 # is_train = True # 是否训练
@@ -164,6 +175,8 @@ upload_data = {
 # is_edbnamelist = False # 自定义特征，对应上面的edbnamelist
 # is_update_eta  = True  # 预测结果上传到eta
 # is_update_report = True # 是否上传报告
+# is_update_warning_data = False  # 是否上传预警数据
+
 
 ### 开关
 is_train = True # 是否训练
@@ -175,7 +188,7 @@ is_edbcode = False # 特征使用edbcoding列表中的
 is_edbnamelist = False # 自定义特征，对应上面的edbnamelist
 is_update_eta  = False  # 预测结果上传到eta
 is_update_report = False # 是否上传报告
-
+is_update_warning_data = True  # 是否上传预警数据
 
 # 数据截取日期
 end_time = '' # 数据截取日期

lib/dataread.py

@@ -114,6 +114,18 @@ def upload_report_data(token, upload_data):
         logger.info("报告上传失败")
         return None
 
+def upload_warning_data(token, warning_data):
+    warning_data = warning_data
+    headers = {"Authorization": token}
+    logger.info("预警上传中...")
+    logger.info(f"token:{token}")
+    logger.info(f"warning_data:{warning_data}" )
+    upload_res = requests.post(url=upload_warning_url, headers=headers, json=warning_data, timeout=(3, 15))
+    if upload_res:
+        return upload_res
+    else:
+        logger.info("预警上传失败")
+        return None
     
 
 # 统计特征频度
@@ -498,12 +510,26 @@ def datachuli(df_zhibiaoshuju,df_zhibiaoliebiao,datecol='date',end_time='',y='y'
     
     logger.info(f'删除预警的特征后数据量：{df.shape}')
 
-    logger.info(f'删除两月不更新特征前数据量：{df.shape}')
+
+    
+    if is_update_warning_data:
+        logger.info(f'上传预警信息')
+        try:
+            warning_data_df = last_update_times_df[last_update_times_df['feature'].isin(columns_to_drop)][['last_update_time','feature']]
+            warning_data_df.sort_values(by='last_update_time',ascending=False,inplace=True)
+            warning_data['data']['WARNING_DATE'] = y_last_update_time
+            warning_data['data']['WARNING_CONTENT'] =  warning_data_df.to_string(index=False)
+            token = get_head_auth_report()
+            upload_warning_data(token, warning_data)
+            logger.info(f'上传预警信息成功')
+        except Exception as e:
+            logger.error(f'上传预警信息失败：{e}')
+            
     # 去掉近最后数据对应的日期在六月以前的列，删除近2月的数据是常熟的列
     current_date = datetime.datetime.now()
     two_months_ago = current_date - timedelta(days=180)
     
-
+    logger.info(f'删除两月不更新特征前数据量：{df.shape}')
     def check_column(col_name):
         '''
         去掉空值列
@@ -673,127 +699,6 @@ def getdata_juxiting(filename, datecol='date',y='y',dataset='',add_kdj=False,is_
 
     return df
 
-# def filter_data(ClassifyName,data):
-#     '''
-#     指标名称保留规则
-#     '''
-    
-#     # 包含 关键词 去除， 返回flase
-#     if any(keyword in data for keyword in ['运费','检修','波动率','地缘政治','股价',
-#                                         '同比','环比','环差','裂差','4WMA','变频','道琼斯','标普500','纳斯达克',
-#                                         '四周均值','名占比','残差','DMA',
-#                                         '连7-连9','4周平均','4周均值','滚动相关性','日本']):
-#         return False
-    
-#     # 检查需要的特征  
-#     # 去掉 分析 分类下的数据
-#     if ClassifyName == '分析':
-#         return False
-    
-#     # 保留 库存中特殊关键词
-#     if ClassifyName == '库存':
-#         if any(keyword in data for keyword in ['原油'  , '美国'  ,'全球'  ,'中国' ,'富查伊拉','ARA' ]):
-#             return True
-#         else:
-#             pass
-#     else:
-#         pass
-    
-#     # 去掉 持仓中不是基金的数据 
-#     if ClassifyName == '持仓':
-#         if '基金' not in data:
-#             return False
-#         else:
-#             pass
-#     else:
-#         pass
-    
-#     # 去掉 航班中不是中国、美国 的数据 
-#     if ClassifyName == '需求':
-#         if '航班' in data :
-#             if '中国' in data or '美国' in data :
-#                 return True
-#             else:
-#                 return False
-#         else:
-#             pass
-#     else:
-#         pass
-    
-#     # 分类为 期货市场，同质性数据取第一个
-#     if ClassifyName == '期货市场':
-#         # 去掉c1-9 以后的
-#         if 'c1-c' in data:
-#             try:
-#                 c = int(data.split('c1-c')[1])
-#             except:
-#                 return False
-#             if c > 9 :
-#                 return False
-#             else:
-#                 pass
-        
-#         else:
-#             pass
-        
-#         # 判断 同质性数据， 字符串开头
-#         strstartdict = {'ICE Brent c':"ICE Brent c14",
-#                     'NYMWX WTI c':"NYMWX WTI c5",
-#                     'INE SC c':"INE SC c1",
-#                     'EFS c':"EFS c",
-#                     'Dubai Swap c':"Dubai Swap c1",
-#                     'Oman Swap c':"Oman Swap c1",
-#                     'DME Oman c':"DME Oman c1",
-#                     'Murban Futures c':"Murban Futures c1",
-#                     'Dubai连合约价格':'Dubai连1合约价格',
-#                     '美国RBOB期货月份合约价格':'美国RBOB期货2309月份合约价格',  
-#                     'Brent连合约价格':'Brent连1合约价格',
-#                     'WTI连合约价格':'WTI连1合约价格',
-#                     '布伦特连合约价格':'Brent连1合约价格',
-#                     'Brent 连合约价格':'Brent连1合约价格',
-#                     'Dubai连合约价格':'Dubai连1合约价格',
-#                     'Brent连':'Brent连1合约价格',
-#                     'brent连':'Brent连1合约价格',
-#                     }
-#         # 判断名称字符串开头是否在 strstartdict.keys中
-#         match = re.match(r'([a-zA-Z\s]+)(\d+)', data)
-#         if match:
-#             part1 = match.group(1)
-#             part2 = match.group(2)
-#             if part1 in [i for i in strstartdict.keys()]:
-#                 if data == strstartdict[part1]:
-#                     return True
-#                 else:
-#                     return False
-#         # data = 'Brent 连7合约价格'
-#         # 判断名称字符串去掉数字后是否在 strstartdict.keys中
-#         match = re.findall(r'\D+',  data)
-#         if match :
-#             if len(match) == 2:
-#                 part1 = match[0]
-#                 part2 = match[1]
-#                 if part1+part2 in [i  for i in strstartdict.keys()]:
-#                     if data == strstartdict[part1+part2]:
-#                         return True
-#                     else:
-#                         return False
-#                 else:
-#                     pass
-#             elif len(match) == 1:
-#                 match = re.findall(r'\D+',  data)
-#             part1 = match[0]
-            
-#             if part1 in [i  for i in strstartdict.keys()]:
-#                 if data == strstartdict[part1]:
-#                     return True
-#                 else:
-#                     return False
-#             else:
-#                 pass
-#         else:
-#             pass
-
-#     return True
 
 def sanitize_filename(filename):  
     # 使用正则表达式替换不合规的字符  

测试环境登录接口调试.ipynb

@@ -0,0 +1,178 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "31c0e11d-c87a-4e95-92a0-d1d09625e255",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from config_jingbo import *\n",
+    "import requests\n",
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "83c81b9e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'http://192.168.100.53:8080/jingbo-dev/api/server/login'"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "login_pushreport_url\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "2b330ee3-c006-4ab1-8558-59c51ac8d86f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'data': {'account': 'api_test',\n",
+       "  'password': 'ZTEwYWRjMzk0OWJhNTlhYmJlNTZlMDU3ZjIwZjg4M2U=',\n",
+       "  'tenantHashCode': '8a4577dbd919675758d57999a1e891fe',\n",
+       "  'terminal': 'API'},\n",
+       " 'funcModule': 'API',\n",
+       " 'funcOperation': '获取token'}"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "login_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dcb6100a-ed2b-4077-a1a9-361c6cb565f9",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'confirmFlg': False, 'data': {'accessToken': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhcGlfdGVzdCIsInRoIjoiOGE0NTc3ZGJkOTE5Njc1NzU4ZDU3OTk5YTFlODkxZmUiLCJsdCI6ImFwaSIsImlzcyI6IiIsInRtIjoiUEMiLCJleHAiOjE3MzE5NDMzODgsImp0aSI6ImMwMTY4NWI1NTFlMzQyNGQ5M2JkZDIyYmE4YzI3YjY5In0.cj5Py63Hl0VML75Yy5VyFz4le-aM3VtFPXjjuwS8Y3w', 'md5Token': 'bd259af3a6d7e158204664387e18c4f5'}, 'status': True}\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "def get_head_auth_report():\n",
+    "    login_res = requests.post(url=login_pushreport_url, json=login_data, timeout=(3, 5))\n",
+    "    text = json.loads(login_res.text)\n",
+    "    print(text)\n",
+    "    if text[\"status\"]:\n",
+    "        token = text[\"data\"][\"accessToken\"]\n",
+    "    return token\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22c0c7c4",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'get_head_auth_report' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[10], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m get_head_auth_report()\n",
+      "\u001b[1;31mNameError\u001b[0m: name 'get_head_auth_report' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "token = get_head_auth_report()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12077ead",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhcGlfdGVzdCIsInRoIjoiOGE0NTc3ZGJkOTE5Njc1NzU4ZDU3OTk5YTFlODkxZmUiLCJsdCI6ImFwaSIsImlzcyI6IiIsInRtIjoiUEMiLCJleHAiOjE3MzE5NDQ1MjMsImp0aSI6IjMxOTFlYTFmMDljOTQ3MjQ5OGIxYmEzMWZiYzUyNWE5In0.XEvtFDbN6Qd9ibTEzLWykcDSgiQQOgBro6vTzXiwbgQ'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a7ae21d1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "warning_data = {'funcModule': '原油特征停更预警', \n",
+    "                'funcOperation': '原油特征停更预警', \n",
+    "                'data': {'WARNING_TYPE_NAME': '日度数据', \n",
+    "                         'WARNING_CONTENT': '''                                    feature last_update_time  update_period warning_date\\n               美国:东海岸地区:炼油厂的投入与使用情况:开工率:当周值       2024-06-14            7.0   2024-06-29\\n                     美国:炼油厂的投入与 使用情况:开工率:当周值       2024-06-14            7.0   2024-06-29\\n               美国:洛基山地区:炼油厂的投入与使用情况:开工率:当周值       2024-06-14            7.0   2024-06-29\\n             \n",
+    " 美国:墨西哥湾沿岸:炼油厂的投入与使用情况:开工率:当周值       2024-06-14            7.0   2024-06-29\\n               美国:西海岸地区:炼油厂的投入与使用情况:开工率:当周值       2024-06-14            7.0   2024-06-29\\n               美国:中西部地区:炼油厂的投入与使用情况:开工率:当周值       2024-06-14            7.0   2024-06-29\\n                                    中国航班执行数       2024-04-10            1.0   2024-04-13\\n                                   美国汽油产量占比       2024-10-04            7.0   2024-10-19\\n                                   美国柴油产量占比       2024-10-04            7.0   2024-10-19\\n                           西北欧即期Brent 211利润       2018-10-24            1.0   2018-10-27\\n                         美国RBOB期货2309月份合约价格       2023-08-31            1.0   2023-09-03\\nPVM Dubai EFS/Swap vs ICE First Month Brent       2023-01-03            1.0   2023-01-06\\n                                   Ebob连1合约       2024-02-28            1.0   2024-03-02\\n                              Brent/迪拜原油升贴水       2022-11-16            1.0   2022-11-19\\n                                  W-B活跃合约价差       2022-01-31            1.0   2022-02-03\\n                                  B-W活跃合约价差       2022-01-31            1.0   2022-02-03\\n                          Brent c2-c3 (结算B)       2024-10-31            1.0   2024-11-03\\n                            WTI c2-c3 (结算B)       2024-10-31            1.0   2024-11-03\\n                               Kpler-海运在途原油       2024-08-20            1.0   2024-08-23\\n                             全球原油库存（剔除富查伊拉）       2024-07-21            7.0   2024-08-05\\n                   原油：山东港口：库存：新口径（周）超季节性/3年       2024-05-24            7.0   2024-06-08\\n                              原油：港口库存：中国（周）       2023-08-04            7.0   2023-08-19\\n                          原油：山东港口：库存：新口径（周）       2024-05-24            7.0   2024-06-08\\n                            美国墨西哥湾87#汽油现货价格       2024-01-31            1.0   2024-02-03\\n                                     欧洲汽油裂解       2024-11-01            1.0   2024-11-04\\n                      Brent c1 02:30收盘价滞后1天       2024-10-30            1.0   2024-11-02\\n                      Brent c2 02:30收盘价滞后1天       2024-10-30            1.0   2024-11-02\\n                        EFS c1 02:30收盘价滞后1天       2024-10-30            1.0   2024-11-02\\n                        EFS c2 02:30收盘价滞后1天       2024-10-30            1.0   2024-11-02\\n                        原油：开工率：东北地区：独立炼厂（周）       2023-03-31            7.0   2023-04-15\\n                                美国墨西哥湾原油钻机数       2024-09-27            7.0   2024-10-12\\n                          Kpler-美国原油净进口（周度）       2024-08-25            7.0   2024-09-09\\n                              中东-也门周度原油海运出口       2023-08-27           21.0   2023-10-09\\n                              中东-巴林周度原油海运出口       2024-04-28            7.0   2024-05-13' ''', \n",
+    " 'WARNING_DATE': '2024-11-14'}}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3e1a9e67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# def upload_warning_data(token, warning_data):\n",
+    "warning_data = warning_data\n",
+    "headers = {\"Authorization\": token}\n",
+    "logger.info(\"预警上传中...\")\n",
+    "logger.info(f\"token:{token}\")\n",
+    "logger.info(f\"warning_data:{warning_data}\" )\n",
+    "upload_res = requests.post(url=upload_warning_url, headers=headers, json=warning_data, timeout=(3, 15))\n",
+    "print(upload_res)\n",
+    "    # if upload_res:\n",
+    "    #     return upload_res\n",
+    "    # else:\n",
+    "    #     logger.info(\"预警上传失败\")\n",
+    "        # return None"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}