dev

core/framework/function/core.php

@@ -1819,6 +1819,42 @@ function wkcache($key, $value, $expire = null)
     return $ret;
 }
 
+function rkcachex($key, $prefix = '')
+{
+    if (C('cache_open')) {
+        $cacher = Cache::getInstance('cacheredis');
+    } else {
+        $cacher = Cache::getInstance('file', null);
+    }
+    if (!$cacher) {
+        throw new Exception('Cannot fetch cache object!');
+    }
+
+    $value = $cacher->get_org($key, $prefix);
+    return $value;
+}
+
+function wkcachex($key, $value, $prefix = '')
+{
+    if (is_array($value)) {
+        throw new Exception('Cannot Support array object!');
+    }
+
+    if (C('cache_open')) {
+        $cacher = Cache::getInstance('cacheredis');
+    } else {
+        $cacher = Cache::getInstance('file', null);
+    }
+    if (!$cacher) {
+        throw new Exception('Cannot fetch cache object!');
+    }
+
+    $ret = $cacher->set_org($key, $value, $prefix);
+    return $ret;
+}
+
+
+
 function incrcache($key,$member,$val,$prefix = '',$def_value = 0)
 {
     if ($key===null || !C('cache_open')) {

crontab/control/minutes.php

@@ -455,14 +455,21 @@ class minutesControl extends BaseCronControl
     private function stat_util()
     {
         //查找最早的充值中的单子
-        $update_earliest_ordertime = function () {
+//        $update_earliest_ordertime = function () {
+//            $mod = Model('refill_detail');
+//            $time = $mod->getEarliestSendTime();
+//            wcache('earliest_sending', ['order_time' => $time], 'refill-stat-');
+//        };
+//        $update_earliest_ordertime();
+
+        $update_earliest_ordertime_bymerchant = function () {
             $mod = Model('refill_detail');
-            $time = $mod->getLatestSendTime();
-            wcache('earliest_sending', ['order_time' => $time], 'refill-stat-');
-            Log::record("earliest_sending order_time={$time}",Log::DEBUG);
-        };
+            $mtimes = $mod->getEarliestSendTimeByMerchant();
+            $val = json_encode($mtimes);
 
-        $update_earliest_ordertime();
+            wkcachex('stat-earliest-ordertime', $val, 'refill-');
+        };
+        $update_earliest_ordertime_bymerchant();
     }
 
     /**

data/model/refill_detail.model.php

@@ -21,7 +21,7 @@ class refill_detailModel extends Model
         return $this->where($condition)->find();
     }
 
-    public function getLatestSendTime()
+    public function getEarliestSendTime()
     {
         $item = $this->field('min(order_time) as send_time')->where(['order_state' => ORDER_STATE_SEND])->find();
         if(empty($item)) {
@@ -30,4 +30,18 @@ class refill_detailModel extends Model
             return intval($item['send_time']);
         }
     }
+
+    public function getEarliestSendTimeByMerchant()
+    {
+        $items = $this->field('mchid,min(order_time) as send_time')->where(['order_state' => ORDER_STATE_SEND])->group('mchid')->select();
+
+        $result = [];
+        foreach($items as $item) {
+            $mchid = intval($item['mchid']);
+            $order_time = intval($item['send_time']);
+            $result[$mchid] = $order_time;
+        }
+
+        return $result;
+    }
 }

plot/refill/MerchantCalc.py

@@ -1,18 +1,14 @@
 
-from .DataStream import EMchPosmap as pos_map, span_days, time_border, calc_interval
+from .DataStream import EMchPosmap as pos_map, span_days
 from .MerchantReader import MerchantReader
-from matplotlib.figure import Figure
-from matplotlib import ticker
-from io import BytesIO
-import numpy as np
-from .algorithm import calc_mchratios, calc_morder_send
-import time as stime
+from .algorithm import calc_morder_lack
+import time as time
 import logging
+import json
 import redis
 
 logger = logging.getLogger('painter')
 
-
 def detail_paths(reader: MerchantReader, tuple_pathes: dict, days: list):
     count = len(days)
 
@@ -28,6 +24,24 @@ def detail_paths(reader: MerchantReader, tuple_pathes: dict, days: list):
             yield mchid, _card_type, _spec, detail_datas
 
 
+def earliest_time(rclient: redis.client):
+    result = {}
+    min_time = int(time.time())
+
+    data = rclient.get('nc_refill-stat-earliest-ordertime')
+    if data is not None:
+        mchid_times = json.loads(data)
+        for mchid,order_time in mchid_times.items():
+            mchid = int(mchid)
+            order_time = int(order_time)
+            result[mchid] = order_time
+            if order_time < min_time:
+                min_time = order_time
+        return result, min_time
+    else:
+        return result, min_time
+
+
 class MerchantCalc(object):
     def __init__(self):
         self._mQuit = False
@@ -50,25 +64,63 @@ class MerchantCalc(object):
             try:
                 if client is None:
                     client = redis_client()
-
-                self._ratios(client)
+                send_amounts = self._send_amounts(client)
+                val = json.dumps({'send_amounts': send_amounts, 'time': int(time.time())})
+                client.set('nc_refill-stat-merchant-sendamount',val)
             except redis.RedisError as ex:
                 logger.error(ex)
             except Exception as ex:
                 logger.error(ex)
+            finally:
+                time.sleep(10)
 
-    def _ratios(self, rclient):
-        def earliest_time():
-            order_time = rclient.hget('nc_refill-stat-earliest_sending', 'order_time')
-            if order_time is not None:
-                return int(order_time)
-            else:
-                return int(stime.time() - 86400)
+    def calc_time(self, start_time: int, end_time: int):
+        reader = MerchantReader()
+        end_time = reader.near_stamp(end_time, False)
+        if end_time is None:
+            raise Exception('end_time data is empty')
+
+        start_time = reader.near_stamp(start_time, True)
+        if start_time is None:
+            raise Exception('start_time data is empty')
+
+        strtime = lambda t: time.strftime('%d-%H:%M:%S', time.localtime(t))
+        logger.debug("near_stamp start_time %s end_time=%s", strtime(start_time), strtime(end_time))
+
+        if start_time >= end_time:
+            raise Exception('start_time equal endtime')
 
-        start_time = earliest_time()
-        end_time = int(stime.time())
-        
-        logger.debug('start_time=%d end_time=%d', start_time, end_time)
+        days = span_days(start_time, end_time)
 
+        strtime = lambda t: time.strftime('%d-%H:%M:%S', time.localtime(t))
+        sdays = [strtime(day) for day in days]
+        logger.debug(sdays)
+
+        return reader,days, start_time, end_time
+        pass
+
+    def _send_amounts(self, rclient):
+        mchid_times,earliest = earliest_time(rclient)
+        end_time = int(time.time())
+        reader, days, start_time, end_time = self.calc_time(earliest, end_time)
+
+        day_stamp = days[0]
+        tuple_pathes = reader.many_tuple_path(days)
+        gen = detail_paths(reader, tuple_pathes, days)
+
+        mamounts = dict()
+        for _mchid, _card_type, _spec, _data in gen:
+            if _mchid not in mchid_times:
+                continue
+            else:
+                _start_time = mchid_times[_mchid]
+
+            send_amounts, lack_amounts = calc_morder_lack(_data, pos_map, start_time - day_stamp, end_time - day_stamp)
+            if _mchid not in mamounts:
+                mamounts[_mchid] = {'send_amounts': send_amounts, 'lack_amounts': lack_amounts}
+            else:
+                mamounts[_mchid]['send_amounts'] += send_amounts
+                mamounts[_mchid]['lack_amounts'] += lack_amounts
+        logger.debug(mamounts)
 
-    pass
+        return mamounts

plot/refill/MerchantReader.py

@@ -51,12 +51,12 @@ class MerchantReader(DataReadStream):
             for name, ls in tuples.items():
                 for tup in ls:
                     _card_type, _spec = tup
-                    if _card_type in card_types:
+                    if card_types is None or _card_type in card_types:
                         if spec is None or _spec == spec:
                             result[name].append((_card_type, _spec))
-
             return result
-        tuples = typespec_filter(tuples,card_types,spec)
+
+        tuples = typespec_filter(tuples, card_types, spec)
         return tuples
 
     def many_tuple_path(self, days: list, mchids: set = None, card_types: set = None, spec: int = None):

plot/refill/algorithm.py

@@ -1,6 +1,9 @@
 
 from .DataStream import EMchPosmap
 import numpy as np
+import logging
+
+logger = logging.getLogger('calcer')
 
 def calc_chratios(data,pos_map,start,end):
     view = data[[pos_map.succ_count, pos_map.fail_count, pos_map.commit_count], :]
@@ -41,3 +44,18 @@ def calc_morder_send(data, pos_map: type(EMchPosmap), start: int, end: int):
 
     return send_count, sums[pos_map.submit_count], sums[pos_map.succ_count], sums[pos_map.fail_count], sums[pos_map.submit_amounts], \
            sums[pos_map.succ_mch_amounts], sums[pos_map.fail_mch_amounts], send_amounts, lack_amounts
+
+
+def calc_morder_lack(data, pos_map: type(EMchPosmap), start: int, end: int):
+    view = data[:, start:end]
+    sums = np.sum(view, axis=1)
+
+    all_return = sums[pos_map.succ_count] + sums[pos_map.fail_count] + 0.0000001
+    ratio = sums[pos_map.succ_count] / all_return
+
+    send_count = sums[pos_map.submit_count] - sums[pos_map.succ_count] - sums[pos_map.fail_count]
+    send_amounts = sums[pos_map.submit_amounts] - sums[pos_map.succ_mch_amounts] - sums[pos_map.fail_mch_amounts]
+    lack_amounts = send_amounts * ratio
+    logger.info("send_count=%d send_amounts=%.4f ratio=%.4f lack_amounts=%.4f", send_count, send_amounts, ratio, lack_amounts)
+
+    return round(send_amounts, 4), round(lack_amounts, 4)

test/TestRefillMonitor.php

@@ -31,4 +31,29 @@ class TestRefillMonitor extends TestCase
             refill\util::monitor_submit(1092, 100, 4, 98.5, $time + $i);
         }
     }
+
+    public function testAddMTimes()
+    {
+        $mtimes = [
+            10299 => 1657831019,
+            10304 => 1657833490,
+            10250 => 1657840392,
+            1093  => 1657848109,
+            10105 => 1657848384,
+            10281 => 1657849018,
+            10201 => 1657849281,
+            10284 => 1657849396,
+            10283 => 1657849577,
+            10264 => 1657849608,
+            10289 => 1657850505,
+            10131 => 1657850569,
+            10222 => 1657850581,
+            10263 => 1657850658,
+            10184 => 1657850674,
+            10305 => 1657850680
+        ];
+
+        $val = json_encode($mtimes);
+        wkcachex('stat-earliest-ordertime', $val, 'refill-');
+    }
 }