dev

plot/app.py

@@ -23,6 +23,7 @@ logger = logging.getLogger('plot')
 from refill import ChannelCumPainter, ChannelCovPainter, ChannelCovSuccPainter
 from refill import MerchantCumRatioPainter, MerchantAmountPainter, MerchantCovRatioPainter
 from refill import filter_chname, filter_cardtype, filter_mchids, get_channels, get_mchids
+from refill import NetcheckCovPainter, get_net_channels
 
 
 def parse_parmeter():
@@ -86,6 +87,7 @@ def ch_covratio():
     except Exception as ex:
         return onError(ex)
 
+
 @app.route('/plot/ch_covsuccs')
 def ch_covsuccs():
     try:
@@ -95,7 +97,7 @@ def ch_covsuccs():
         chnames = filter_chname(chnames)
 
         painter = ChannelCovSuccPainter(start_time=start_time, end_time=end_time, chnames=chnames, card_types=card_types, spec=spec,
-                                    filter_wave=filter_wave)
+                                        filter_wave=filter_wave)
         buf, ratios = painter.paint()
         data = base64.b64encode(buf.getbuffer()).decode("ascii")
 
@@ -104,12 +106,27 @@ def ch_covsuccs():
         return onError(ex)
 
 
+@app.route('/plot/net_check')
+def net_check():
+    try:
+        logger.debug('start net_check')
+        start_time, end_time, card_types, spec, filter_wave = parse_parmeter()
+        chnames = request.args.get('chnames')
+        chnames = filter_chname(chnames)
+
+        painter = NetcheckCovPainter(start_time=start_time, end_time=end_time, chnames=chnames, filter_wave=filter_wave)
+        buf, ratios = painter.paint()
+        data = base64.b64encode(buf.getbuffer()).decode("ascii")
+
+        return jsonify({'img': data, 'ratios': ratios, 'state': 'success'})
+    except Exception as ex:
+        return onError(ex)
 
 
-@app.route('/plot/channels')
-def channels():
+@app.route('/plot/net_channels')
+def net_channels():
     try:
-        channels = get_channels()
+        channels = get_net_channels()
         return jsonify(channels)
     except Exception as ex:
         logger.error(ex)

plot/refill/NetcheckCovPainter.py

@@ -0,0 +1,62 @@
+from .DataStream import ENetPosmap as pos_map
+from .NetcheckPainter import NetcheckPainter, net_pathes
+from .NetchkReader import NetchkReader
+from matplotlib.figure import Figure
+from matplotlib import ticker
+from io import BytesIO
+import numpy as np
+from .algorithm import calc_cov_netfail
+import time as time
+
+import logging
+logger = logging.getLogger('ChannelCumPainter')
+
+class NetcheckCovPainter(NetcheckPainter):
+    def __init__(self, start_time: int, end_time: int, chnames: set = None, filter_wave: int = None):
+        self._reader = NetchkReader()
+        if filter_wave is None:
+            filter_wave = 3600
+        self._chnames, self._filter_wave = chnames, filter_wave
+        _days, self._start_time, self._end_time, self._interval = self.calc_time(self._reader, start_time, end_time)
+        pass
+
+    def paint(self):
+        reader = self._reader
+        self._days, _start_time, _end_time, _interval = self.calc_time(self._reader, self._start_time - self._filter_wave, self._end_time)
+
+        tuple_pathes = reader.many_tuple_path(self._days, self._chnames)
+        gen = net_pathes(reader, tuple_pathes, self._days)
+        if len(self._days) == 0:
+            return BytesIO()
+
+        day_stamp = self._days[0]
+        fig_create, fig_flush = self._fig_funs()
+        ax, fig = fig_create()
+        x = np.array([d - self._start_time for d in range(self._start_time, self._end_time)])
+        window = np.ones(self._filter_wave)
+
+        left_len = self._start_time - _start_time
+        right_len = _end_time - self._end_time
+
+        stime = lambda t: time.strftime('%d-%H:%M:%S', time.localtime(t))
+        logger.debug("near_stamp start_time %d, %s end_time=%s left_len=%d right_len=%d",
+                     self._start_time, stime(self._start_time), stime(self._end_time),left_len,right_len)
+
+        chname_ratios = []
+        for _chname, _data in gen:
+            fail, fails, count = calc_cov_netfail(_data, pos_map, _start_time - day_stamp, _end_time - day_stamp, window, left_len,right_len)
+            label, ratio = self._label(chname=_chname, succ=fails, count=count)
+            ax.plot(x, fail, ls='-', label=label)
+            if _chname != 'all':
+                chname_ratios.append((_chname, ratio))
+
+        ticks = [d - self._start_time for d in range(self._start_time, self._end_time + 1, self._interval)]
+        xlables = [time.strftime('%d-%H:%M:%S', time.localtime(d + self._start_time)) for d in ticks]
+        buf = fig_flush(ax, fig, ticks, xlables)
+
+        chname_ratios = sorted(chname_ratios, key=lambda x: (x[1], x[0]), reverse=True)
+        result = []
+        for name, ratio in chname_ratios:
+            result.append(f'{name}:{ratio}')
+
+        return buf, result

plot/refill/NetcheckPainter.py

@@ -0,0 +1,80 @@
+from .DataStream import ENetPosmap as pos_map, day_stamp, span_days, time_border, calc_interval
+from .NetchkReader import NetchkReader
+from .PainterBase import PainterBase
+from matplotlib.figure import Figure
+from matplotlib import ticker
+from io import BytesIO
+import numpy as np
+import time as time
+
+import logging
+logger = logging.getLogger('ChannelPainter')
+
+_all_net_channels = set()
+def _add_net_channel(channel):
+    if channel not in _all_net_channels:
+        _all_net_channels.add(channel)
+
+def get_net_channels():
+    logger.debug(_all_net_channels)
+    return list(_all_net_channels)
+
+def net_pathes(reader: NetchkReader, names: set, days: list):
+    count = len(days)
+    if len(names) > 1:
+        _all_data = reader.init_data(count)
+    else:
+        _all_data = None
+
+    for name in names:
+        _add_net_channel(name)
+        _ch_data = reader.init_data(count)
+
+        for i, day in enumerate(days):
+            data = reader.read(day, name)
+            if data is not None:
+                column_pos = i * 86400
+                view = _ch_data[:, column_pos:column_pos + 86400]
+                view += data
+
+        if _all_data is not None:
+            _all_data += _ch_data
+        yield name, _ch_data
+
+    if _all_data is not None:
+        yield 'all', _all_data
+
+
+class NetcheckPainter(PainterBase):
+    def _fig_funs(self):
+        def create():
+            fig = Figure(figsize=(19, 8))
+            ax = fig.subplots()
+            ax.set_title('success ratio')
+            ax.set(xlabel='time', ylabel='ratio')
+            return ax, fig
+
+        def flush(ax, fig, ticks, lables):
+            ax.yaxis.set_major_formatter(ticker.PercentFormatter(xmax=1, decimals=4))
+            ax.set_xticks(ticks=ticks)
+            ax.set_xticklabels(lables)
+            fig.autofmt_xdate()
+            ax.grid()
+            fig.subplots_adjust(left=0.1, right=0.8, top=0.95, bottom=0.1)
+            ax.legend(bbox_to_anchor=(1, 1), loc='upper left')
+
+            buf = BytesIO()
+            fig.savefig(buf, format="png")
+            return buf
+
+        return create, flush
+
+    def _label(self, chname, succ, count):
+        lable = f"{chname}"
+        if count > 0:
+            ratio = round(succ * 100 / count, 5)
+        else:
+            ratio = 0.00
+        lable += f":{succ}/{count}={ratio}%"
+
+        return lable, ratio

plot/refill/NetchkReader.py

@@ -1,5 +1,10 @@
 from .DataStream import DataReadStream,open_hdf5
+from .DataStream import ENetPosmap as pos_map
+
 import numpy as np
+import re
+from collections import defaultdict
+import time as time
 
 __all__ = ['NetchkReader']
 
@@ -16,6 +21,44 @@ class NetchkReader(DataReadStream):
         self.close()
         super(NetchkReader, self).__del__()
 
-    def read(self, path):
-        pass
+    def tuple_path(self, day: int, chnames: set = None):
+        def parse(path):
+            items = re.split(r'/', path)
+            (_prifix, _version, today, chname) = items
+            return chname
+
+        all = True if chnames is None or len(chnames) == 0 else False
+        channels = set()
+        pathes = self.datasets(day)
+        for path in pathes:
+            _chname = parse(path)
+            if all:
+                channels.add(_chname)
+            elif _chname in chnames:
+                channels.add(_chname)
+            else:
+                continue
+
+        return channels
+
+    def many_tuple_path(self, days: list, chnames: set = None):
+        all = set()
+        for day in days:
+            channels = self.tuple_path(day, chnames)
+            all = all.union(channels)
+        return all
+
+    def init_data(self, days):
+        dim = pos_map.dim()
+        return np.zeros((dim, 86400 * days))
+
+    def read(self, today, chname):
+        path = f'/{self._version}/{today}/{chname}'
+
+        hfive = self.file
+        if path in hfive:
+            return hfive[path]
+        else:
+            return None
+    pass
     pass

plot/refill/__init__.py

@@ -17,12 +17,18 @@ from .MerchantPainter import get_mchids
 from .MerchantCumRatioPainter import MerchantCumRatioPainter
 from .MerchantCovRatioPainter import MerchantCovRatioPainter
 from .MerchantAmountPainter import MerchantAmountPainter
+
+from .NetcheckPainter import get_net_channels
+from .NetcheckCovPainter import NetcheckCovPainter
 from .helper import filter_chname, filter_cardtype, filter_mchids
 from .MAmountCalc import MAmountCalc
 from .MProfitRatioCalc import MProfitRatioCalc
 from .MTimesRatioCalc import MTimesRatioCalc
 from .ChSpeedRatioCalc import ChSpeedRatioCalc
+
 from .WriterConsumer import WriterConsumer
+
+
 from .server_util import opt_parse
 
 __all__ = ['DataWriteStream', 'DataReadStream',
@@ -35,4 +41,6 @@ __all__ = ['DataWriteStream', 'DataReadStream',
            'filter_chname', 'filter_cardtype', 'filter_mchids',
            'MAmountCalc', 'MProfitRatioCalc', 'MTimesRatioCalc',
            'ChSpeedRatioCalc',
+           'NetcheckCovPainter',
+           'get_net_channels',
            'opt_parse']

plot/refill/algorithm.py

@@ -177,3 +177,21 @@ def calc_mch_profit(data, pos_map: type(EMchPosmap), start: int, end: int):
     profit = mch_amounts - ch_amounts
 
     return int(submit_count), int(succ_count), int(fail_count), round(succ_ratio, 5), round(profit, 3)
+
+def calc_cov_netfail(data, pos_map, start, end, window, left_len,right_len):
+    view = data[[pos_map.succ_count, pos_map.fail_count], :]
+    view = view[:, start:end]
+
+    sums = np.sum(view, axis=1)
+    succs = int(sums[0])
+    fails = int(sums[1])
+
+    succ = view[0, :]
+    fail = view[1, :]
+    succ   = np.convolve(succ, window, 'same')
+    fail = np.convolve(fail, window, 'same')
+    succ   = succ[left_len:end - start - right_len]
+    fail = fail[left_len:end - start - right_len]
+
+    fail = fail / (fail + succ + 0.0000001)
+    return fail, fails, (succs + fails)

plot/testPlot.py

@@ -91,6 +91,26 @@ class MyTestCase(unittest.TestCase):
         painter = MerchantAmountPainter(start_time=start_time, end_time=end_time, mchids=set(), card_types={4, 5, 6})
         painter.paint()
 
+    def test_netcheck(self):
+        from refill import NetchkReader
+        import time
+        from refill import day_stamp
+
+        day = day_stamp(int(time.time()) - 4 * 86400)
+        days = [day,day-86400]
+        
+        net = NetchkReader()
+        channels = net.tuple_path(day)
+        channels = net.many_tuple_path(days)
+
+    def test_net_painter(self):
+        from refill import NetcheckCovPainter
+
+        start_time = int(time.time()) - 5 * 86400
+        end_time = int(time.time()) - 4 * 86400
+        painter = NetcheckCovPainter(start_time=start_time, end_time=end_time)
+        painter.paint()
+
     def testDays(self):
         from refill import MerchantReader
         from refill import ChannelReader