refactor mamount.py for stat

docker/compose/homecuda/statcalc/docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - /mnt/shoplog:/var/www/html/data/log
       - /mnt/stdata:/var/www/html/data/stdata
     container_name: "panda-mamount"
-    command: ['python','mamount.py', '-h', '192.168.3.104', '-p', '6379']
+    command: ['python','mamount_lack.py', '-h', '192.168.3.104', '-p', '6379']
 
   mpratios:
     image: pycpu:3.7.10

docker/compose/workcuda/statcalc/docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - /mnt/shoplog:/var/www/html/data/log
       - /mnt/stdata:/var/www/html/data/stdata
     container_name: "panda-mamount"
-    command: [ 'python','mamount.py', '-h', '192.168.3.46', '-p', '6379' ]
+    command: [ 'python','mamount_lack.py', '-h', '192.168.3.46', '-p', '6379' ]
 
   mpratios:
     image: pycpu:3.7.10

docker/compose/xyz/statcalc/docker-compose.yml

@@ -9,7 +9,7 @@ services:
       - /mnt/shoplog:/var/www/html/data/log
       - /mnt/stdata:/var/www/html/data/stdata
     container_name: "panda-mamount"
-    command: ['python','mamount.py', '-h', '172.26.105.125', '-p', '6379']
+    command: ['python','mamount_lack.py', '-h', '172.26.105.125', '-p', '6379']
 
   mpratios:
     image: pycpu:3.7.10

docker/compose/yl/ylstatcalc/docker-compose.yml

@@ -8,8 +8,8 @@ services:
       - ../conf/etc/localtime:/etc/localtime:ro
       - /mnt/yllog:/var/www/html/data/log
       - /mnt/stdata:/var/www/html/data/stdata
-    container_name: "panda-mamount"
-    command: ['python','mamount.py', '-h', '172.26.80.7', '-p', '6379']
+    container_name: "yl-mamount"
+    command: ['python','mamount_lack.py', '-h', '172.26.80.7', '-p', '6379']
 
 
   mpratios:
@@ -19,5 +19,5 @@ services:
       - ../conf/etc/localtime:/etc/localtime:ro
       - /mnt/yllog:/var/www/html/data/log
       - /mnt/stdata:/var/www/html/data/stdata
-    container_name: "panda-mpratios"
+    container_name: "yl-mpratios"
     command: ['python','mprofit_ratio.py', '-h', '172.26.80.7', '-p', '6379']

plot/mamount.py

@@ -1,32 +0,0 @@
-
-from refill import MerchantCalc
-import signal as sig
-import sys,getopt
-
-import logging
-logging.basicConfig(filename='/var/www/html/data/log/statcalc.log',
-                    format='%(levelname)10s  %(asctime)s  %(name)10s %(thread)d %(message)s',
-                    level=logging.DEBUG)
-
-
-if __name__ == '__main__':
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "h:p:", ["host=", 'port='])
-    except Exception as ex:
-        print(ex)
-        sys.exit(2)
-
-    rhost = ''
-    rport=6379
-    for o, val in opts:
-        if o in ("-h", "--host"):
-            rhost = val
-        elif o in ('-p', "--port"):
-            rport = int(val)
-        else:
-            print("Err argv")
-
-    calc = MerchantCalc()
-    calc.set_redis(rhost,rport)
-    sig.signal(sig.SIGINT, lambda: calc.stop())
-    calc.run()

plot/mamount_lack.py

@@ -0,0 +1,19 @@
+from refill import opt_parse, MAmountCalc
+import signal as sig
+import sys,getopt
+
+import logging
+logging.basicConfig(filename='/var/www/html/data/log/statcalc.log',
+                    format='%(levelname)10s  %(asctime)s  %(name)10s %(thread)d %(message)s',
+                    level=logging.DEBUG)
+logger = logging.getLogger('mamount')
+
+if __name__ == '__main__':
+    try:
+        rhost, rport = opt_parse()
+        calc = MAmountCalc()
+        calc.set_redis(rhost, rport)
+        sig.signal(sig.SIGINT, lambda: calc.stop())
+        calc.run()
+    except Exception as ex:
+        logger.error(ex)

plot/refill/MAmountCalc.py

@@ -0,0 +1,61 @@
+from .MerchantCalcBase import MerchantCalcBase, mch_detail_paths, mch_paths, detail_paths
+from .algorithm import calc_morder_lack
+from .DataStream import EMchPosmap as pos_map
+import time as time
+import json
+import redis
+
+import logging
+logger = logging.getLogger('mch_amount_calc')
+
+def earliest_time(rclient: redis.client):
+    result = {}
+    min_time = int(time.time())
+
+    data = rclient.get('nc_refill-stat-earliest-ordertime')
+    if data is not None:
+        mchid_times = json.loads(data)
+        for mchid,order_time in mchid_times.items():
+            mchid = int(mchid)
+            order_time = int(order_time)
+            result[mchid] = order_time
+            if order_time < min_time:
+                min_time = order_time
+        return result, min_time
+    else:
+        return result, min_time
+
+class MAmountCalc(MerchantCalcBase):
+    def _calc_handler(self, rclient):
+        mchid_times,earliest = earliest_time(rclient)
+        end_time = int(time.time())
+        reader = self._reader()
+
+        days, start_time, end_time = self.calc_time(reader, earliest, end_time)
+        day_stamp = days[0]
+        tuple_pathes = reader.many_tuple_path(days)
+        gen = detail_paths(reader, tuple_pathes, days)
+
+        mamounts = dict()
+        for _mchid, _card_type, _spec, _data in gen:
+            if _mchid not in mchid_times:
+                continue
+            else:
+                _start_time = mchid_times[_mchid]
+
+            send_amounts, lack_amounts = calc_morder_lack(_data, pos_map, start_time - day_stamp, end_time - day_stamp)
+            if _mchid not in mamounts:
+                mamounts[_mchid] = {'send_amounts': send_amounts, 'lack_amounts': lack_amounts}
+            else:
+                mamounts[_mchid]['send_amounts'] += send_amounts
+                mamounts[_mchid]['lack_amounts'] += lack_amounts
+
+        result = dict()
+        for _mchid, _val in mamounts.items():
+            _send_amounts = round(_val['send_amounts'], 2)
+            _lack_amounts = round(_val['lack_amounts'], 2)
+            result[_mchid] = {'send_amounts': _send_amounts, 'lack_amounts': _lack_amounts}
+
+        val = json.dumps({'send_amounts': result, 'time': int(time.time())})
+        rclient.set('nc_refill-stat-merchant-sendamount',val)
+        logger.debug(result)

plot/refill/MerchantCalc.py

@@ -1,151 +0,0 @@
-from redis.client import Redis
-
-from .DataStream import EMchPosmap as pos_map, span_days
-from .MerchantReader import MerchantReader
-from .algorithm import calc_morder_lack
-import time as time
-import logging
-import json
-import redis
-
-logger = logging.getLogger('merchantcalc')
-
-def detail_paths(reader: MerchantReader, tuple_pathes: dict, days: list):
-    count = len(days)
-
-    for mchid, tup in tuple_pathes.items():
-        for _card_type, _spec in tup:
-            detail_datas = reader.init_data(count)
-            for i, day in enumerate(days):
-                data = reader.read(day, mchid, _card_type, _spec)
-                if data is not None:
-                    column_pos = i * 86400
-                    view = detail_datas[:, column_pos:column_pos + 86400]
-                    view += data
-            yield mchid, _card_type, _spec, detail_datas
-
-
-def earliest_time(rclient: redis.client):
-    result = {}
-    min_time = int(time.time())
-
-    data = rclient.get('nc_refill-stat-earliest-ordertime')
-    if data is not None:
-        mchid_times = json.loads(data)
-        for mchid,order_time in mchid_times.items():
-            mchid = int(mchid)
-            order_time = int(order_time)
-            result[mchid] = order_time
-            if order_time < min_time:
-                min_time = order_time
-        return result, min_time
-    else:
-        return result, min_time
-
-class MerchantCalc(object):
-    def __init__(self):
-        self._mQuit = False
-        self._mRHost = ''
-        self._mRPort = 6379
-        pass
-
-    def set_redis(self, rhost, rport):
-        self._mRHost = rhost
-        self._mRPort = rport
-
-    def run(self):
-        def redis_client():
-            pool = redis.ConnectionPool(host=self._mRHost, port=self._mRPort, db=0)
-            client = redis.Redis(connection_pool=pool)
-            return client
-
-        def calc_send_amounts(client):
-            send_amounts = self._send_amounts(client)
-            val = json.dumps({'send_amounts': send_amounts, 'time': int(time.time())})
-            client.set('nc_refill-stat-merchant-sendamount',val)
-            pass
-
-        def calc_profit_ratio(client):
-            profit_ratio = self._profit_ratio(client)
-            pass
-
-        client = None
-        loop = 0;
-        while self._mQuit == False:
-            try:
-                if client is None:
-                    client = redis_client()
-
-                #每10秒计算一次欠费金额
-                if loop % 10 == 0:
-                    calc_send_amounts(client)
-
-                # if loop % 2 == 0:
-                #     calc_profit_ratio(client)
-                #     pass
-
-            except redis.RedisError as ex:
-                logger.error(ex)
-            except Exception as ex:
-                logger.error(ex)
-            finally:
-                time.sleep(1)
-                loop += 1
-
-
-    def calc_time(self, start_time: int, end_time: int):
-        reader = MerchantReader()
-        end_time = reader.near_stamp(end_time, False)
-        if end_time is None:
-            raise Exception('end_time data is empty')
-
-        start_time = reader.near_stamp(start_time, True)
-        if start_time is None:
-            raise Exception('start_time data is empty')
-
-        strtime = lambda t: time.strftime('%d-%H:%M:%S', time.localtime(t))
-        logger.debug("near_stamp start_time %s end_time=%s", strtime(start_time), strtime(end_time))
-
-        if start_time >= end_time:
-            raise Exception('start_time equal endtime')
-
-        days = span_days(start_time, end_time)
-
-        strtime = lambda t: time.strftime('%d-%H:%M:%S', time.localtime(t))
-        sdays = [strtime(day) for day in days]
-        logger.debug(sdays)
-
-        return reader, days, start_time, end_time
-        pass
-
-    def _send_amounts(self, rclient):
-        mchid_times,earliest = earliest_time(rclient)
-        end_time = int(time.time())
-        reader, days, start_time, end_time = self.calc_time(earliest, end_time)
-
-        day_stamp = days[0]
-        tuple_pathes = reader.many_tuple_path(days)
-        gen = detail_paths(reader, tuple_pathes, days)
-
-        mamounts = dict()
-        for _mchid, _card_type, _spec, _data in gen:
-            if _mchid not in mchid_times:
-                continue
-            else:
-                _start_time = mchid_times[_mchid]
-
-            send_amounts, lack_amounts = calc_morder_lack(_data, pos_map, start_time - day_stamp, end_time - day_stamp)
-            if _mchid not in mamounts:
-                mamounts[_mchid] = {'send_amounts': send_amounts, 'lack_amounts': lack_amounts}
-            else:
-                mamounts[_mchid]['send_amounts'] += send_amounts
-                mamounts[_mchid]['lack_amounts'] += lack_amounts
-
-        result = dict()
-        for _mchid, _val in mamounts.items():
-            _send_amounts = round(_val['send_amounts'], 2)
-            _lack_amounts = round(_val['lack_amounts'], 2)
-            result[_mchid] = {'send_amounts': _send_amounts, 'lack_amounts': _lack_amounts}
-
-        logger.debug(result)
-        return result

plot/refill/__init__.py

@@ -11,7 +11,7 @@ from .ChannelReader import ChannelReader
 from .ChannelPainter import ChannelPainter,get_channels
 from .MerchantPainter import MerchantPainter,get_mchids
 from .helper import filter_chname, filter_cardtype, filter_mchids
-from .MerchantCalc import MerchantCalc
+from .MAmountCalc import MAmountCalc
 from .MProfitRatioCalc import MProfitRatioCalc
 from .WriterConsumer import WriterConsumer
 from .server_util import opt_parse
@@ -22,5 +22,5 @@ __all__ = ['DataWriteStream', 'DataReadStream',
            'ChannelPainter', 'MerchantPainter', 'get_channels', 'get_mchids',
            'queueListener', 'open_hdf5', 'day_stamp', 'time_border',
            'filter_chname', 'filter_cardtype', 'filter_mchids',
-           'MerchantCalc','MProfitRatioCalc'
+           'MAmountCalc','MProfitRatioCalc'
            'opt_parse']

plot/thdf5.py

@@ -169,8 +169,8 @@ class DataTest(unittest.TestCase):
         queueListener.prepare_data()
 
     def test_mamount(self):
-        from refill import MerchantCalc
-        calc = MerchantCalc()
+        from refill import MAmountCalc
+        calc = MAmountCalc()
         calc.set_redis(self.__redis_host,'6379')
         calc.run()