Merge branch 'raccount' of 39.97.239.116:gyfl/xyzshop into raccount

docker/compose/workcuda/cli/docker-compose.yml

@@ -40,9 +40,7 @@ services:
     command: ['vender-init']
 
   pythoncli:
-    image: pyflask:3.7.10
-#    ports:
-#      - 5000:5000
+    image: pycpu:3.7.10
     volumes:
       - /mnt/xyzshop/docker/compose/workcuda/conf/etc/localtime:/etc/localtime:ro
       - /mnt/xyzshop:/var/www/html
@@ -50,4 +48,16 @@ services:
       - /mnt/stdata:/var/www/html/data/stdata
     container_name: "panda-python"
     extra_hosts:
+      - "docker.hostip:172.17.0.1"
+
+  pybash:
+    image: pycpu:3.7.10
+    volumes:
+      - /mnt/xyzshop/docker/compose/workcuda/conf/etc/localtime:/etc/localtime:ro
+      - /mnt/xyzshop:/var/www/html
+      - /mnt/shoplog:/var/www/html/data/log
+      - /mnt/stdata:/var/www/html/data/stdata
+    container_name: "panda-pybash"
+    command: [python,'/var/www/html/plot/copy_channel.py']
+    extra_hosts:
       - "docker.hostip:172.17.0.1"

docker/compose/yl/ylcli/docker-compose.yml

@@ -27,4 +27,18 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '8'
+          cpus: '8'
+
+  pybash:
+    image: pycpu:3.7.10
+    volumes:
+      - ../../../../:/var/www/html
+      - ../conf/etc/localtime:/etc/localtime:ro
+      - ../conf/php/php-swoole.ini:/usr/local/etc/php/php.ini
+      - /nfs/ylupload:/var/www/html/data/upload
+      - /mnt/yllog:/var/www/html/data/log
+      - /mnt/stdata:/var/www/html/data/stdata
+    container_name: "yl-pybash"
+    command: [python,'/var/www/html/plot/copy_channel.py']
+    extra_hosts:
+      - "docker.hostip:172.17.0.1"

plot/copy_channel.py

@@ -0,0 +1,31 @@
+from refill import queueListener, ChannelReader, ChannelWriter, open_hdf5, mktime,day_stamp
+from refill import MerchantReader,MerchantWriter
+
+class CopyChannel(object):
+    def copy(self):
+        def copy(reader,writer,paths):
+            for path in paths:
+                data = reader.read_path(path)
+                writer.write_set(path,data)
+            pass
+
+        latest_time = mktime('2023-03-01 00:00:00')
+        lastest_day = day_stamp(latest_time)
+
+        reader = ChannelReader()
+        hDFive = open_hdf5('/var/www/html/data/stdata/channel_bak.hdf5', True)
+        writer = ChannelWriter(hDFive)
+        days = reader.days()
+        for day in days:
+            day = int(day)
+            if day < lastest_day:
+                continue
+            paths = reader.datasets(day)
+            copy(reader,writer,paths)
+            print(paths)
+
+        hDFive.close()
+        pass
+
+copier = CopyChannel()
+copier.copy()

plot/refill/ChannelReader.py

@@ -128,7 +128,7 @@ class ChannelReader(DataReadStream):
             for name, ls in tuples.items():
                 for tup in ls:
                     _card_type, _spec = tup
-                    if _card_type in card_types:
+                    if card_types is None or _card_type in card_types:
                         if spec is None or _spec == spec:
                             result[name].append((_card_type, _spec))
 
@@ -171,4 +171,21 @@ class ChannelReader(DataReadStream):
                 return result
         else:
             return None
+
+    def read_path(self, path):
+        hfive = self.file
+        if path in hfive:
+            dset = hfive[path]
+            if dset.shape[0] == pos_map.dim():
+                return dset
+            else:
+                dim = pos_map.dim()
+                result = np.zeros((dim, 86400))
+                result[0:dset.shape[0], :] = dset[:, :]
+                return result
+        else:
+            return None
+
+
+
     pass

plot/refill/ChannelWriter.py

@@ -147,3 +147,16 @@ class ChannelWriter(DataWriteStream):
         else:
             dset = hfive[path]
         return dset
+
+    def write_set(self,path,data):
+        if data is None:
+            return False
+
+        dset = self._data_set(path)
+        dset[:, :] = data
+
+        self.file.flush()
+        return True
+
+
+

plot/refill/MerchantReader.py

@@ -97,4 +97,18 @@ class MerchantReader(DataReadStream):
             return hfive[path]
         else:
             return None
+
+    def read_path(self, path):
+        hfive = self.file
+        if path in hfive:
+            dset = hfive[path]
+            if dset.shape[0] == pos_map.dim():
+                return dset
+            else:
+                dim = pos_map.dim()
+                result = np.zeros((dim, 86400))
+                result[0:dset.shape[0], :] = dset[:, :]
+                return result
+        else:
+            return None
     pass

plot/refill/MerchantWriter.py

@@ -137,4 +137,14 @@ class MerchantWriter(DataWriteStream):
             hfive.flush()
         else:
             dset = hfive[path]
-        return dset
+        return dset
+
+    def write_set(self,path,data):
+        if data is None:
+            return False
+
+        dset = self._data_set(path)
+        dset = data
+
+        self.file.flush()
+        return True

plot/refill/__init__.py

@@ -1,5 +1,5 @@
 from .QueueListener import queueListener
-from .DataStream import DataWriteStream, DataReadStream, open_hdf5, day_stamp, time_border
+from .DataStream import DataWriteStream, DataReadStream, open_hdf5, day_stamp, time_border,mktime
 from .MerchantWriter import MerchantWriter
 from .ChannelWriter import ChannelWriter
 from .NetchkWriter import NetchkWriter
@@ -41,7 +41,7 @@ __all__ = ['DataWriteStream', 'DataReadStream',
            'ChannelCumPainter', 'ChannelCovPainter', 'ChannelCovSuccPainter', 'ChannelSpeedAnalyzePainter',
            'MerchantCumRatioPainter', 'MerchantAmountPainter', 'MerchantCovRatioPainter',
            'get_channels', 'get_mchids',
-           'queueListener', 'open_hdf5', 'day_stamp', 'time_border',
+           'queueListener', 'open_hdf5', 'day_stamp', 'time_border','mktime'
            'filter_chname', 'filter_cardtype', 'filter_mchids',
            'MAmountCalc', 'MProfitRatioCalc', 'MTimesRatioCalc',
            'ChSpeedRatioCalc',

plot/testCommand.py

@@ -0,0 +1,56 @@
+import unittest
+from refill import queueListener, ChannelReader, ChannelWriter, open_hdf5, mktime,day_stamp
+from refill import MerchantReader,MerchantWriter
+import signal as sig
+import sys,getopt
+import logging
+
+class TestCommand(unittest.TestCase):
+    def test_copy_channel(self):
+        def copy(reader,writer,paths):
+            for path in paths:
+                data = reader.read_path(path)
+                writer.write_set(path,data)
+            pass
+
+        latest_time = mktime('2023-03-01 00:00:00')
+        lastest_day = day_stamp(latest_time)
+
+        reader = ChannelReader()
+        hDFive = open_hdf5('/var/www/html/data/stdata/channel_bak.hdf5', True)
+        writer = ChannelWriter(hDFive)
+        days = reader.days()
+        for day in days:
+            day = int(day)
+            if day < lastest_day:
+                continue
+            paths = reader.datasets(day)
+            copy(reader,writer,paths)
+            print(paths)
+        pass
+
+    def test_copy_mch(self):
+        def copy(reader,writer,paths):
+            for path in paths:
+                data = reader.read_path(path)
+                writer.write_set(path,data)
+            pass
+
+        latest_time = mktime('2023-03-01 00:00:00')
+        lastest_day = day_stamp(latest_time)
+
+        reader = MerchantReader()
+        hDFive = open_hdf5('/var/www/html/data/stdata/merchant_bak.hdf5', True)
+        writer = MerchantWriter(hDFive)
+        days = reader.days()
+        for day in days:
+            day = int(day)
+            if day < lastest_day:
+                continue
+            paths = reader.datasets(day)
+            copy(reader,writer,paths)
+            print(paths)
+        pass
+
+if __name__ == '__main__':
+    unittest.main()

plot/thdf5.py

@@ -15,17 +15,19 @@ logging.basicConfig(filename='/var/www/html/data/log/plot_test.log',
 logger = logging.getLogger('test_log')
 
 class DataTest(unittest.TestCase):
-    __redis_host = '192.168.3.104'
-    # __redis_host = '192.168.3.46'
+    # __redis_host = '192.168.3.104'
+    __redis_host = '192.168.3.226'
 
     def test_connect(self):
-        pool = redis.ConnectionPool(host='121.89.223.81', port=57649, db=0)
+        pool = redis.ConnectionPool(host=self.__redis_host, port=6379, db=0)
         r = redis.Redis(connection_pool=pool)
         for item in r.hscan_iter('nc_channel_monitor_commit'):
             key = item[0]
             val = item[1]
             print(str(key, encoding="utf-8"), str(val, encoding="utf-8"))
 
+
+
     def test_regex(self):
         text = 'succm-lingzh-1-4-30-1618291260'
         result = re.split(r'-', text)