dev

plot/plot_control.py

@@ -12,46 +12,73 @@ import signal as sig
 import sys, getopt
 
 from refill import ChannelPainter
-from refill import filter_chname,filter_cardtype
+from refill import filter_chname,filter_cardtype,filter_mchids
 
 logging.basicConfig(filename='/var/www/html/data/log/flask.log',
                     format='%(levelname)10s  %(asctime)s  %(name)10s %(thread)d %(message)s',
                     level=logging.DEBUG)
 logger = logging.getLogger('plot')
 
-@app.route('/plot/ch_ratio')
-def ch_ratio():
-    try:
-        logger.debug('start chratio')
-        cur_time = int(time.time())
+def parse_parmeter():
+    logger.debug('start chratio')
+    cur_time = int(time.time())
+
+    start_time = request.args.get('start_time')
+    start_time = cur_time - 7200 if start_time is None else int(start_time.strip())
+
+    end_time = request.args.get('end_time')
+    end_time = cur_time if end_time is None else int(end_time.strip())
+
+    stime = lambda t: time.strftime('%d-%H:%M:%S', time.localtime(t))
+    logger.debug("start_time=%s end_time=%s",stime(start_time) ,stime(end_time))
 
-        start_time = request.args.get('start_time')
-        start_time = cur_time - 7200 if start_time is None else int(start_time.strip())
+    card_types = request.args.get('card_types')
+    card_types = filter_cardtype(card_types)
 
-        end_time = request.args.get('end_time')
-        end_time = cur_time if end_time is None else int(end_time.strip())
+    spec = request.args.get('spec')
+    if spec is not None:
+        spec = int(spec.strip())
 
-        stime = lambda t: time.strftime('%d-%H:%M:%S', time.localtime(t))
-        logger.debug("start_time=%s end_time=%s",stime(start_time) ,stime(end_time))
+    filter_wave = request.args.get('filter_wave')
+    if filter_wave is not None:
+        filter_wave = int(filter_wave.strip())
 
+    return start_time,end_time,card_types,spec,filter_wave
+
+
+@app.route('/plot/ch_ratio')
+def ch_ratio():
+    try:
+        start_time, end_time, card_types, spec, filter_wave = parse_parmeter()
         chnames = request.args.get('chnames')
         chnames = filter_chname(chnames)
 
-        card_types = request.args.get('card_types')
-        card_types = filter_cardtype(card_types)
+        painter = ChannelPainter(start_time=start_time, end_time=end_time, chnames=chnames, card_types=card_types, spec=spec, filter_wave=filter_wave)
+        buf = painter.paint()
+        data = base64.b64encode(buf.getbuffer()).decode("ascii")
 
-        spec = request.args.get('spec')
-        if spec is not None:
-                spec = int(spec.strip())
+        return f"<img src='data:image/png;base64,{data}'/>"
+    except Exception as ex:
+        logger.error(ex)
+        data = base64.b64encode(b'').decode("ascii")
+        return f"<img src='data:image/png;base64,{data}'/>"
 
-        painter = ChannelPainter(start_time=start_time, end_time=end_time, chnames=chnames, card_types=card_types, spec=spec)
-        buf = painter.paint()
+@app.route('/plot/mch_ratio')
+def mch_ratio():
+    try:
+        start_time, end_time, card_types, spec, filter_wave = parse_parmeter()
+        mchids = request.args.get('mchids')
+        mchids = filter_mchids(mchids)
 
+        painter = ChannelPainter(start_time=start_time, end_time=end_time, chnames=mchids, card_types=card_types, spec=spec, filter_wave=filter_wave)
+        buf = painter.paint()
         data = base64.b64encode(buf.getbuffer()).decode("ascii")
+
         return f"<img src='data:image/png;base64,{data}'/>"
     except Exception as ex:
         logger.error(ex)
-        return f"<img src='data:image/png;base64,{[]}'/>"
+        data = base64.b64encode(b'').decode("ascii")
+        return f"<img src='data:image/png;base64,{data}'/>"
 
 
 if __name__ == "__main__":

plot/refill/ChannelPainter.py

@@ -51,9 +51,9 @@ def allpathes(reader: ChannelReader, tuple_pathes: dict, days: list, spec=None):
 
 
 class ChannelPainter(object):
-    def __init__(self, start_time: int, end_time: int, chnames: set = None, card_types: set = None, spec: int = None):
+    def __init__(self, start_time: int, end_time: int, chnames: set = None, card_types: set = None, spec: int = None, filter_wave: int = None):
         self._reader = ChannelReader()
-        _start_time, _end_time, self._chnames, self._card_types, self._spec = start_time, end_time, chnames, card_types, spec
+        _start_time, _end_time, self._chnames, self._card_types, self._spec, self._filter_wave = start_time, end_time, chnames, card_types, spec, filter_wave
 
         start_time = self._reader.near_stamp(_start_time,True)
         end_time = self._reader.near_stamp(_end_time,False)
@@ -98,18 +98,21 @@ class ChannelPainter(object):
         tuple_pathes = reader.many_tuple_path(self._days, self._chnames, self._card_types, self._spec)
         gen = allpathes(self._reader, tuple_pathes, self._days, self._spec)
         if len(self._days) == 0:
-            return []
+            return BytesIO()
 
         day_stamp = self._days[0]
         fig_create, fig_flush = self._fig_funs()
         ax, fig = fig_create()
         x = np.array([d - self._start_time for d in range(self._start_time, self._end_time)])
 
-        window = np.ones(600) / 600
+        if self._filter_wave is not None and self._filter_wave > 1:
+            window = np.ones(self._filter_wave) / self._filter_wave
+        else:
+            window = None
+
         for _chname, _card_type, _spec, _data in gen:
             succ, count, y = calc_chratios(_data, pos_map, self._start_time - day_stamp, self._end_time - day_stamp)
-            print(_chname)
-            y = np.convolve(y, window, 'same')
+            y = np.convolve(y, window, 'same') if window is not None else y
             ax.plot(x, y, ls='-', label=self._label(chname=_chname, succ=succ, count=count, card_type=_card_type, spec=_spec))
 
         ticks = [d - self._start_time for d in range(self._start_time, self._end_time + 1, self._interval)]

plot/refill/ChannelReader.py

@@ -5,7 +5,7 @@ import numpy as np
 import re
 from collections import defaultdict
 
-__all__ = ['MerchantReader']
+__all__ = ['ChannelReader']
 
 import logging
 
@@ -89,5 +89,4 @@ class ChannelReader(DataReadStream):
             return hfive[path]
         else:
             return None
-
-    pass
+    pass

plot/refill/MerchantPainter.py

@@ -0,0 +1,149 @@
+from .DataStream import EMchPosmap as pos_map, day_stamp, span_days, time_border, calc_interval
+from .MerchantReader import MerchantReader
+from matplotlib.figure import Figure
+from matplotlib import ticker
+from io import BytesIO
+import numpy as np
+from .algorithm import calc_mchratios
+import time as time
+
+import logging
+logger = logging.getLogger('painter')
+
+def allpathes(reader: MerchantReader, tuple_pathes: dict, days: list, spec=None):
+    count = len(days)
+    show_detail = True if len(list(tuple_pathes.keys())) == 1 else False
+    if show_detail == False:
+        all_datas = reader.init_data(count)
+    else:
+        all_datas = None
+
+    for name, tup in tuple_pathes.items():
+        mch_datas = reader.init_data(count)
+        for _card_type, _spec in tup:
+            if spec is not None and _spec != spec:
+                continue
+
+            if show_detail:
+                detail_datas = reader.init_data(count)
+            else:
+                detail_datas = None
+
+            for i, day in enumerate(days):
+                data = reader.read(day, name, _card_type, _spec)
+                if data is not None:
+                    column_pos = i * 86400
+                    view = mch_datas[:, column_pos:column_pos + 86400]
+                    view += data
+
+                    if show_detail:
+                        view = detail_datas[:, column_pos:column_pos + 86400]
+                        view += data
+            if show_detail:
+                yield name, _card_type, _spec, detail_datas
+        if all_datas is not None:
+            all_datas += mch_datas
+        yield name, None, None, mch_datas
+
+    if show_detail == False:
+        yield 'all', None, None, all_datas
+
+
+class MerchantPainter(object):
+    def __init__(self, start_time: int, end_time: int, mchids: set = None, card_types: set = None, spec: int = None, filter_wave: int = None):
+        self._reader = MerchantReader()
+        _start_time, _end_time, self._mchids, self._card_types, self._spec, self._filter_wave = start_time, end_time, mchids, card_types, spec, filter_wave
+
+        start_time = self._reader.near_stamp(_start_time,True)
+        end_time = self._reader.near_stamp(_end_time,False)
+
+        stime = lambda t: time.strftime('%d-%H:%M:%S', time.localtime(t))
+        logger.debug("start_time=%s end_time=%s",stime(start_time) ,stime(end_time))
+
+        interval = calc_interval(start_time, end_time)
+        start_time = time_border(interval, start_time, True)
+        end_time = time_border(interval, end_time, False)
+        self._days = span_days(start_time, end_time)
+        self._start_time = start_time
+        self._end_time = end_time
+        self._interval = interval
+        pass
+
+    def _fig_funs(self):
+        def create():
+            fig = Figure(figsize=(19, 8))
+            ax = fig.subplots()
+            ax.set_title('success ratio')
+            ax.set(xlabel='time', ylabel='ratio')
+            return ax, fig
+
+        def flush(ax, fig, ticks, lables):
+            ax.yaxis.set_major_formatter(ticker.PercentFormatter(xmax=1, decimals=4))
+            ax.set_xticks(ticks=ticks)
+            ax.set_xticklabels(lables)
+            fig.autofmt_xdate()
+            ax.grid()
+            fig.subplots_adjust(left=0.1, right=0.8, top=0.95, bottom=0.1)
+            ax.legend(bbox_to_anchor=(1, 1), loc='upper left')
+
+            buf = BytesIO()
+            fig.savefig(buf, format="png")
+            return buf
+
+        return create, flush
+
+    def paint(self):
+        tuple_pathes = self._reader.many_tuple_path(self._days, self._mchids, self._card_types, self._spec)
+        gen = allpathes(self._reader, tuple_pathes, self._days, self._spec)
+        if len(self._days) == 0:
+            return BytesIO()
+
+        day_stamp = self._days[0]
+        fig_create, fig_flush = self._fig_funs()
+        ax, fig = fig_create()
+        x = np.array([d - self._start_time for d in range(self._start_time, self._end_time)])
+
+        if self._filter_wave is not None and self._filter_wave > 1:
+            window = np.ones(self._filter_wave) / self._filter_wave
+        else:
+            window = None
+
+        for _mchid, _card_type, _spec, _data in gen:
+            succ, count, y = calc_mchratios(_data, pos_map, self._start_time - day_stamp, self._end_time - day_stamp)
+            y = np.convolve(y, window, 'same') if window is not None else y
+            ax.plot(x, y, ls='-', label=self._label(chname=_mchid, succ=succ, count=count, card_type=_card_type, spec=_spec))
+
+        ticks = [d - self._start_time for d in range(self._start_time, self._end_time + 1, self._interval)]
+        xlables = [time.strftime('%d-%H:%M:%S', time.localtime(d + self._start_time)) for d in ticks]
+
+        return fig_flush(ax, fig, ticks, xlables)
+
+    def _label(self, chname, succ, count, card_type=None, spec=None):
+        _card_type = None
+        if card_type == 1:
+            _card_type = 'SY'
+        elif card_type == 2:
+            _card_type = 'SH'
+        elif card_type == 4:
+            _card_type = 'YD'
+        elif card_type == 5:
+            _card_type = 'LT'
+        elif card_type == 6:
+            _card_type = 'DX'
+        elif card_type == 7:
+            _card_type = 'TH'
+
+        lable = f"{chname}"
+        if _card_type is not None:
+            lable += f"-{_card_type}"
+
+        if spec is not None:
+            lable += f"-{spec}"
+
+        if count > 0:
+            ratio = round(succ * 100 / count, 2)
+        else:
+            ratio = 0.00
+        lable += f":{succ}/{count}={ratio}%"
+
+        return lable

plot/refill/MerchantReader.py

@@ -2,6 +2,8 @@
 from .DataStream import DataReadStream, day_stamp,open_hdf5
 from .DataStream import EMchPosmap as pos_map
 import numpy as np
+import re
+from collections import defaultdict
 
 __all__ = ['MerchantReader']
 
@@ -19,6 +21,70 @@ class MerchantReader(DataReadStream):
         super(MerchantReader, self).__del__()
         pass
 
-    def read(self, path):
-        pass
+    def tuple_path(self, day: int, mchids: set = None, card_types: set = None, spec: int = None):
+        def parse(path):
+            items = re.split(r'/', path)
+            (_prifix, _version, today, mchid, card_type, spec) = items
+            return int(mchid), int(card_type), int(spec)
+
+        tuples = defaultdict(list)
+        pathes = self.datasets(day)
+        for path in pathes:
+            _mchid, _card_type, _spec = parse(path)
+            tuples[_mchid].append((_card_type, _spec))
+
+        def name_filter(tuples, mchids):
+            all = True if mchids is None or len(mchids) == 0 else False
+            if all:
+                return tuples
+            else:
+                result = defaultdict(list)
+                for mchid, tup in tuples.items():
+                    if mchid in mchids:
+                        result[mchid] = tup
+                return result
+
+        tuples = name_filter(tuples, mchids)
+
+        def typespec_filter(tuples, card_types, spec):
+            result = defaultdict(list)
+            for name, ls in tuples.items():
+                for tup in ls:
+                    _card_type, _spec = tup
+                    if _card_type in card_types:
+                        if spec is None or _spec == spec:
+                            result[name].append((_card_type, _spec))
+
+            return result
+        tuples = typespec_filter(tuples,card_types,spec)
+        return tuples
+
+    def many_tuple_path(self, days: list, mchids: set = None, card_types: set = None, spec: int = None):
+        def merge(l,r):
+            for mchid,ls in l.items():
+                if mchid in r:
+                    r[mchid].union(set(ls))
+                else:
+                    r[mchid] = set(ls)
+            return r
+
+        all = defaultdict(set)
+        for day in days:
+            tuples = self.tuple_path(day, mchids, card_types, spec)
+            all = merge(tuples,all)
+
+        return all
+
+    def init_data(self, days):
+        dim = pos_map.dim()
+        return np.zeros((dim, 86400 * days))
+
+    def read(self, day_stamp, mchid, card_type, spec):
+        path = f'/{self._version}/{day_stamp}/{mchid}/{card_type}/{spec}'
+
+        hfive = self.file
+        if path in hfive:
+            return hfive[path]
+        else:
+            return None
     pass

plot/refill/__init__.py

@@ -9,12 +9,13 @@ from .NetchkReader import NetchkReader
 from .ChannelReader import ChannelReader
 
 from .ChannelPainter import ChannelPainter
+from .MerchantPainter import MerchantPainter
 from .helper import filter_chname, filter_cardtype
 
 __all__ = ['DataWriteStream', 'DataReadStream',
            'MerchantWriter', 'ChannelWriter', 'NetchkWriter',
            'MerchantReader', 'NetchkReader', 'ChannelReader',
-           'ChannelPainter',
+           'ChannelPainter', 'MerchantPainter',
            'queueListener', 'open_hdf5', 'day_stamp', 'time_border',
-           'filter_chname', 'filter_cardtype'
+           'filter_chname', 'filter_cardtype','filter_mchids'
            ]

plot/refill/algorithm.py

@@ -12,3 +12,16 @@ def calc_chratios(data,pos_map,start,end):
     y = succ / commit
     y = y.ravel()
     return int(all[0, -1]), int(all[0, -1] + all[1, -1]), y
+
+
+def calc_mchratios(data,pos_map,start,end):
+    view = data[[pos_map.succ_count, pos_map.fail_count, pos_map.submit_count], :]
+    view = view[:, start:end]
+
+    all = np.cumsum(view, axis=1)
+    succ = all[0, :]
+    commit = all[0, :] + all[1, :]
+    commit += 0.0000001
+    y = succ / commit
+    y = y.ravel()
+    return int(all[0, -1]), int(all[0, -1] + all[1, -1]), y

plot/refill/helper.py

@@ -20,6 +20,22 @@ def filter_chname(chnames):
         chnames = set(namelist)
     return chnames
 
+def filter_mchids(mchids):
+    if mchids is not None:
+        mchids = mchids.strip()
+
+    if mchids is None or len(mchids) == 0:
+        mchids = set()
+    else:
+        items = re.split(r',', mchids)
+        names = [name.strip() for name in items]
+        namelist = []
+        for name in names:
+            if len(name) > 0:
+                namelist.append(int(name))
+        mchids = set(namelist)
+    return mchids
+
 def filter_cardtype(card_types):
     if card_types is not None:
         card_types = card_types.strip()