添加获取数据并记录停更预警信息

lib/dataread.py

@@ -209,6 +209,7 @@ def create_feature_last_update_time(df):
     return last_update_times_df,y_last_update_time
 
 
+
 # 统计特征频度
 def featurePindu(dataset):
     # 读取文件
@@ -1115,6 +1116,8 @@ class EtaReader():
             df3.rename(columns={'Value': EdbName}, inplace=True)
             # 将数据存储df1
             df = pd.merge(df, df3, how='outer',on='DataTime',suffixes= ('', '_y'))
+            # 按时间排序
+            df = df.sort_values(by='DataTime', ascending=True)
             return df
                 
         else:
@@ -1144,7 +1147,7 @@ class EtaReader():
         '''
 
         # 构建新的DataFrame  df  df1
-        df = pd.DataFrame(columns=['指标分类', '指标名称', '指标id', '频度','指标来源','来源id'])
+        df = pd.DataFrame(columns=['指标分类', '指标名称', '指标id', '频度','指标来源','来源id','最后更新时间','更新周期','预警日期'])
         df1 = pd.DataFrame(columns=['DataTime'])
 
 
@@ -1207,12 +1210,38 @@ class EtaReader():
                         isSave = self.filter_yuanyou_data(ClassifyName,EdbName)
                         if isSave:
                             # 保存到df
+                            df1 = self.edbcodegetdata(df1,EdbCode,EdbName)
+                            # 取df1所有行最后一列
+                            edbname_df = df1[['DataTime',f'{EdbName}']]
+                            edbname_df = edbname_df.dropna()
+                            
+                            if len(edbname_df) == 0:
+                                logger.info(f'指标名称：{EdbName} 没有数据')
+                                continue
+                            try:
+                                time_sequence = edbname_df['DataTime'].values.tolist()[-10:]
+                            except IndexError:
+                                time_sequence = edbname_df['DataTime'].values.tolist()
+                            # 使用Counter来统计每个星期几的出现次数
+                            from collections import Counter
+                            weekday_counter = Counter(datetime.datetime.strptime(time_str, "%Y-%m-%d").strftime('%A') for time_str in time_sequence)
+
+                            # 打印出现次数最多的星期几
+                            try:
+                                most_common_weekday = weekday_counter.most_common(1)[0][0]
+                                # 计算两周后的日期
+                                warning_date = (datetime.datetime.strptime(time_sequence[-1], "%Y-%m-%d") + datetime.timedelta(weeks=2)).strftime("%Y-%m-%d")
+                            except IndexError:
+                                most_common_weekday = '其他'
+                            if '日' in Frequency:
+                                most_common_weekday = '每天'
+                                warning_date = (datetime.datetime.strptime(time_sequence[-1], "%Y-%m-%d") + datetime.timedelta(days=3)).strftime("%Y-%m-%d")
+
                             # 保存频度  指标名称  分类  指标id 到 df
-                            df2 = pd.DataFrame({'指标分类': ClassifyName, '指标名称': EdbName, '指标id': EdbCode, '频度': Frequency,'指标来源':SourceName,'来源id':Source},index=[0])
+                            df2 = pd.DataFrame({'指标分类': ClassifyName, '指标名称': EdbName, '指标id': EdbCode, '频度': Frequency,'指标来源':SourceName,'来源id':Source,'最后更新时间':edbname_df['DataTime'].values[-1],'更新周期':most_common_weekday,'预警日期':warning_date},index=[0])
                             
                             # df = pd.merge(df, df2, how='outer') 
                             df = pd.concat([df, df2])
-                            df1 = self.edbcodegetdata(df1,EdbCode,EdbName)
                         else:
                             logger.info(f'跳过指标 {EdbName}')
 

lib/tools.py

@@ -411,7 +411,6 @@ class SQLiteHandler:
 
     def update_data(self, table_name, set_values, where_condition):
         query = f"UPDATE {table_name} SET {set_values} WHERE {where_condition}"
-        logger.info('更新数据sql'+ query)
         self.execute_query(query)
         self.commit()