xyzshop/test/TestRefillClear.php

<?php


use PHPUnit\Framework\TestCase;

define('APP_ID', 'test');
define('BASE_ROOT_PATH', str_replace('/test', '', dirname(__FILE__)));

require_once(BASE_ROOT_PATH . '/global.php');
require_once(BASE_CORE_PATH . '/lrlz.php');
require_once(BASE_ROOT_PATH . '/fooder.php');

class TestRefillClear extends TestCase
{
    public static function setUpBeforeClass(): void
    {
        Base::run_util();
    }

    //docker-compose run phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::testClearFirstDay)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function testClearFirstDay()
    {
        $start_date = strtotime($this->earlist_orderday());
        $end_date   = $start_date + 86400;

        $this->clear($start_date,$end_date);
    }
    //2021年，8、9、10、11

    //docker-compose run -d phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::testDecember20)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function testDecember20()
    {
        $start_date = strtotime('2021-12-20');
        $end_date   = strtotime('2022-12-28');
        $this->clear($start_date,$end_date);
    }

    //docker-compose run -d phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::test202201)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function test202201()
    {
        $start_date = strtotime('2022-01-10');
        $end_date   = strtotime('2022-01-16');
        $this->clear($start_date,$end_date);
    }

    public function testAllRecords()
    {
        $rorder_getter = function ($mchid,$mch_order,$mod_refill)
        {
            $result = [];
            $i = 0;
            while(true)
            {
                $start = $i * 1000;
                $ritems = $mod_refill->table('refill_order')->field('*')
                    ->where(['mchid' => $mchid, 'mch_order' => $mch_order])
                    ->order('order_id desc')
                    ->limit("{$start},1000")->master(true)->select();
                $i++;

                $result = array_merge($ritems,$result);
                if(count($ritems) < 1000) {
                    break;
                }
            }

            return $result;
        };

        $vorder_getter = function ($order_ids,$mod_vr)
        {
            $result = [];
            $i = 0;
            while(true)
            {
                $start = $i * 1000;

                $vitems = $mod_vr->table('vr_order')->field('*')
                    ->where(['order_id' => ['in', $order_ids]])
                    ->order('order_id asc')
                    ->limit("{$start},1000")->master(true)->select();
                $i++;

                $result = array_merge($vitems,$result);
                if(count($vitems) < 1000) {
                    break;
                }
            }

            return $result;
        };

        $mod_refill = Model('refill_order');
        $ritems = $rorder_getter(10234,'2021120700113026892979',$mod_refill);
        $oids = [];
        foreach ($ritems as $item) {
            $oids[] = $item['order_id'];
        }

        $mod_vr = Model('vr_order');
        $vitems = $vorder_getter($oids,$mod_vr);

        $ret = count($vitems) == count($ritems);
    }

    private function clear($start_date, $end_date,$line = 0)
    {
        $order_finder = function ($start,$fp)
        {
            $end = $start + 3600;
            $cond = ['order_time&order_time' => ['_multi' => true, ['egt', $start], ['lt', $end]],
                     'inner_status' => 0];

            $i = 0;
            while (true)
            {
                $start = $i * 1000;
                $items = Model()->table('refill_order')
                    ->field('mchid,mch_order,order_time')
                    ->where($cond)
                    ->order('order_time desc')->limit("{$start},1000")->select();
                $i++;

                if(empty($items)) break;
                foreach ($items as $item) {
                    fputcsv($fp,$item);
                }
            }
        };

        $order_saver = function ($start,$file) use ($order_finder)
        {
            if(file_exists($file)) {
                return $file;
            }

            $fp = fopen($file,'w+');
            if($fp === false) {
                Log::record("Cannot open file {$file}.",Log::ERR);
                return false;
            }

            $end = $start + 86400;
            for ($hour = $start; $hour < $end; $hour += 3600) {
                $order_finder($hour,$fp);
            }
            fclose($fp);

            return $file;
        };

        $handle_days = function ($start_date, $end_date,$line = 0) use ($order_saver)
        {
            for ($date = $start_date; $date < $end_date; $date += 86400)
            {
                $sdate = date('Y-m-d',$date);

                $morder_file = BASE_DATA_PATH . "/log/order/{$sdate}-morder.csv";
                $check_file  = BASE_DATA_PATH . "/log/order/{$sdate}-check.csv";
                $refill_file  = BASE_DATA_PATH . "/log/order/{$sdate}-refill.csv";
                $vr_file  = BASE_DATA_PATH . "/log/order/{$sdate}-vr.csv";

                $file = $order_saver($date,$morder_file);
                if($file === false) {
                    Log::record("order_saver cannot open morder file.",Log::ERR);
                    break;
                }

                $this->order_delete($file,$check_file,$refill_file,$vr_file,$line);
            }
        };

        $handle_days($start_date, $end_date,$line);
    }

    private function order_delete($file,$check_file,$refill_file,$vr_file,$line)
    {
        $rorder_getter = function ($mchid,$mch_order,$mod_refill)
        {
            $result = [];
            $i = 0;
            while(true)
            {
                $start = $i * 1000;
                $ritems = $mod_refill->table('refill_order')->field('*')
                    ->where(['mchid' => $mchid, 'mch_order' => $mch_order])
                    ->order('order_id desc')
                    ->limit("{$start},1000")->master(true)->select();
                $i++;

                $result = array_merge($ritems,$result);
                if(count($ritems) < 1000) {
                    break;
                }
            }

            return $result;
        };

        $vorder_getter = function ($order_ids,$mod_vr)
        {
            $result = [];
            $i = 0;
            while(true)
            {
                $start = $i * 1000;

                $vitems = $mod_vr->table('vr_order')->field('*')
                    ->where(['order_id' => ['in', $order_ids]])
                    ->order('order_id asc')
                    ->limit("{$start},1000")->master(true)->select();
                $i++;

                $result = array_merge($vitems,$result);
                if(count($vitems) < 1000) {
                    break;
                }
            }

            return $result;
        };


        $order_checker = function ($mchid,$mch_order,$frefill,$fvr) use($rorder_getter,$vorder_getter)
        {
            $mod_refill = Model('refill_order');
            $ritems = $rorder_getter($mchid,$mch_order,$mod_refill);

            if(empty($ritems)) {
                return [false,'没查到记录.'];
            }
            elseif(count($ritems) === 1) {
                return [true,[]];
            }

            $oids = [];
            $inner_state = [0 => 0 ,1 => 1];
            foreach ($ritems as $item) {
                $oids[] = $item['order_id'];
                $state = intval($item['inner_status']);
                $inner_state[$state] += 1;
            }

            //检查inner_status=0唯一。
            if($inner_state[0] != 1) {
                return [false,"inner_status=0 counts = {$inner_state[0]}"];
            }

            $mod_vr = Model('vr_order');
            $vitems = $vorder_getter($oids,$mod_vr);

            //检查refill 表记录和vr_order表记录是否一致
            if(count($ritems) !== count($vitems)) {
                return [false,'refill_orders != vr_orders'];
            }

            $order_state = [];
            foreach ($vitems as $item)
            {
                $state = intval($item['order_state']);
                if(!array_key_exists($state,$order_state)) {
                    $order_state[$state] = 0;
                }

                $order_state[$state] += 1;
            }

            //检查一种订单只能有，成功或者失败状态.
            $vcount = count($vitems);
            if($vcount != $order_state[ORDER_STATE_SUCCESS] + $order_state[ORDER_STATE_CANCEL]) {
                return [false,'ORDER_STATE_SUCCESS + ORDER_STATE_CANCEL != order count.'];
            }
            elseif($order_state[ORDER_STATE_SUCCESS] > 1) {
                //成功订单只能小于等于1，否则为错误
                return [false,'ORDER_STATE_SUCCESS count > 1'];
            }
            else {
                foreach ($ritems as $item) {
                    fputcsv($frefill,$item);
                }
                foreach ($vitems as $item) {
                    fputcsv($fvr,$item);
                }
            }

            $inner_orders = [];
            foreach ($ritems as $item)
            {
                $status = intval($item['inner_status']);
                if($status === 1) {
                    $inner_orders[] = $item['order_id'];
                }
            }

            return [true,$inner_orders];
        };

        $delter = function ($order_ids,$order_time)
        {
            $mod_refill = Model('refill_order');
            $mod_refill->table('refill_order')->where(['order_id' => ['in',$order_ids],'order_time' => $order_time])->delete();
            $mod_vr = Model('vr_order');
            $mod_vr->table('vr_order')->where(['order_id' => ['in',$order_ids]])->delete();
        };

        $position = function ($fp,$line)
        {
            $index = 0;
            while(!feof($fp) && $index < $line) {
                $items = fgetcsv($fp);
                $index += 1;
            }
        };


        $fp = fopen($file,'r');
        if($fp === false) {
            return false;
        }

        $fcheck = fopen($check_file,'a+');
        if($fcheck === false) {
            fclose($fp);
            return false;
        }
        $frefill = fopen($refill_file,'a+');
        $fvr = fopen($vr_file,'a+');

        $position($fp,$line);
        while(!feof($fp))
        {
            [$mchid, $mch_order, $order_time] = fgetcsv($fp);

            if(empty($mchid) || empty($mch_order)) continue;

            [$succ,$err] = $order_checker($mchid,$mch_order,$frefill,$fvr);
            if($succ === false) {
                fputcsv($fcheck,[$mchid,$mch_order,$err]);
            }
            else
            {
                $order_ids = $err;
                if(!empty($order_ids)) {
                    $delter($order_ids,$order_time);
                }
            }
        }

        fclose($fp);
        fclose($fcheck);
        fclose($frefill);
        fclose($fvr);
    }

    private function earlist_orderday()
    {
        $mod_refill = Model('refill_order');
        $item = $mod_refill->table('refill_order')->field('order_time')->order('order_id asc')->find();
        if(empty($item)) {
            return false;
        } else {
            $order_time = intval($item['order_time']);
            $day_stamp = date('Y-m-d',$order_time);

            return $day_stamp;
        }
    }

    public function testEarlist()
    {
        $time = $this->earlist_orderday();
    }

    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    /// 以下代码为清除pdlog
//    const pdlog_table_name = 'pd_log_bak';
    const pdlog_table_name = 'pd_log';
    //docker-compose run -d phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::testLogSeventh)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function testLogSeventh()
    {
        $this->delete_log(74224997,strtotime('2021-09-01'));
    }

    //docker-compose run -d phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::testLogNine)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function testLogNine()
    {
        $this->delete_log(127113184,strtotime('2021-10-01'));
    }

    //docker-compose run -d phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::testLogTen)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function testLogTen()
    {
        $this->delete_log(306132968,strtotime('2021-11-01'));
    }

    //docker-compose run -d phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::testLogEleven)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function testLogEleven()
    {
        $this->delete_log(619026072,strtotime('2021-12-01'));
    }

    //docker-compose run -d phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::testLogDec20)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function testLogDec20()
    {
        $this->delete_log(1079138574,strtotime('2021-12-20'));
    }

    //docker-compose run -d phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::testLogDec29)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function testLogDec29()
    {
        $this->delete_log(2147483648,strtotime('2022-01-01'));
    }

    private function delete_log($start_id,$endtime)
    {
        //需要的函数
        $lgid_getter = function ()
        {
            $lgid = rkcache('lastest_lgid');
            $lgid = intval($lgid);
            return $lgid;
        };

        $lgid_writter = function ($lgid)
        {
            wkcache('lastest_lgid',$lgid);
        };

        $maxid_getter = function () {
            $items = Model()->table(self::pdlog_table_name)->field('max(lg_id) as maxid')->select();
            return intval($items['0']['maxid']);
        };
        ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
        $mod_refill = Model('refill_order');
        $pd_log = Model();
        ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
        /// 一条一条的删除日志
        $finder = function ($start_id, $endtime, $maxid)
        {
            $start = $start_id;
            $quit = false;

            while (!$quit)
            {
                if($start >= $maxid) break;

                $end =  $start + 1000;
                $cond = ['lg_id' => [['egt', $start], ['lt',$end], 'and']];

                $items = Model()->table(self::pdlog_table_name)
                    ->field('*')
                    ->where($cond)
                    ->order('lg_id asc')->select();

                foreach ($items as $item)
                {
                    $add_time = intval($item['lg_add_time']);
                    if($add_time >= $endtime){
                        $quit = true;
                        break;
                    }
                    yield $item;
                }

                $start = $end;
            }
        };

        $checker = function ($log) use ($mod_refill)
        {
            $sdate = date('Y-m-d H:i:s',$log['lg_add_time']);
            Log::record("{$sdate} lgid={$log['lg_id']},type={$log['lg_type']},sn={$log['lg_order_sn']}",Log::DEBUG);

            $logid = intval($log['lg_id']);
            $lgtype = $log['lg_type'];
            if (in_array($lgtype, ['bonus_add_money','hand_out_bonus', 'bonus_refund', 'bonus_expire'])) {
                return [$logid,false];
            }

            $order_sn = $log['lg_order_sn'];
            if(empty($order_sn) && strlen($order_sn) < 16) {
                return [false,false];
            }

            if (!in_array($lgtype, ['order_pay', 'order_freeze', 'order_cancel'])) {
                return [false,false];
            }

            $order = $mod_refill->table('refill_order')->field('*')->where(['order_sn' => $order_sn])->master(true)->find();
            if(empty($order)) {
                return [$logid,true];
            } else {
                return [false,false];
            }
        };

        $delter = function ($logid) use ($pd_log)
        {
            if ($logid > 0) {
                $result = $pd_log->table(self::pdlog_table_name)->where(['lg_id' => $logid])->delete();
                return $result;
            } else {
                return false;
            }
        };

        $onebyone_handler = function ($start_id,$endtime,$maxid,$fLog) use($finder,$checker,$delter)
        {
            $logs = $finder($start_id,$endtime,$maxid);
            foreach ($logs as $item)
            {
                [$logid,$save] = $checker($item);
                if($logid !== false) {
                    $delter($logid);
                }
                if($save) {
                    fputcsv($fLog,$item);
                }
            }
        };

        ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
        /// 一批一批的删除日志
        $batch_finder = function ($start_id, $endtime, $maxid)
        {
            $start = $start_id;
            $quit = false;
            while (!$quit)
            {
                if($start >= $maxid) break;

                $end =  $start + 200;
                $cond = ['lg_id' => [['egt', $start], ['lt',$end], 'and']];

                $items = Model()->table(self::pdlog_table_name)
                    ->field('*')
                    ->where($cond)
                    ->order('lg_id asc')->select();

                $logs = [];
                foreach ($items as $item)
                {
                    $add_time = intval($item['lg_add_time']);
                    if($add_time >= $endtime){
                        $quit = true;
                        break;
                    }
                    else {
                        $logs[] = $item;
                    }
                }

                if(!empty($logs)) {
                    yield $logs;
                }

                $start = $end;
            }
        };

        //return：1,可以直接删除，不需要存储; 2,不可以删除，3，检查后可以删除
        $batch_check_log = function ($log)
        {
            $sdate = date('Y-m-d H:i:s',$log['lg_add_time']);
            Log::record("{$sdate} lgid={$log['lg_id']},type={$log['lg_type']},sn={$log['lg_order_sn']}",Log::DEBUG);

            $logid = intval($log['lg_id']);
            $order_sn = $log['lg_order_sn'];
            $lgtype = $log['lg_type'];

            if (in_array($lgtype, ['bonus_add_money','hand_out_bonus', 'bonus_refund', 'bonus_expire'])) {
                return [1,$logid,$order_sn];
            }

            if(empty($order_sn) && strlen($order_sn) < 16) {
                return [2,false,false];
            }

            if (!in_array($lgtype, ['order_pay', 'order_freeze', 'order_cancel'])) {
                return [2,false,false];
            }

            return [3,$logid,$order_sn];
        };

        $batch_checker = function ($logs) use ($mod_refill,$batch_check_log)
        {
            $del_ids = [];
            $savers = [];

            $exist_logs = [];
            $sns = [];
            foreach ($logs as $log)
            {
                [$type,$logid,$order_sn] = $batch_check_log($log);
                if($type === 1) {
                    $del_ids[] = $logid;
                }
                elseif($type === 3) {
                    $exist_logs[$logid] = $log;
                    $sns[] = $order_sn;
                }
                else {

                }
            }

            if(!empty($sns))
            {
                $sns = array_unique($sns);
                $orders = $mod_refill->table('refill_order')->field('order_sn')->where(['order_sn' => ['in', $sns]])->master(true)->select();

                $sn_sns = [];
                foreach ($orders as $item) {
                    $sn = $item['order_sn'];
                    $sn_sns[$sn] = $sn;
                }

                foreach ($exist_logs as $logid => $log)
                {
                    $sn = $log['lg_order_sn'];
                    if(!array_key_exists($sn,$sn_sns)) {
                        $del_ids[] = $logid;
                        $savers[] = $log;
                    }
                }
            }

            return [$del_ids,$savers];
        };

        $batch_delter = function ($logids) use ($pd_log)
        {
            if (!empty($logids)) {
                $result = $pd_log->table(self::pdlog_table_name)->where(['lg_id' => ['in',$logids]])->delete();
                return $result;
            } else {
                return false;
            }
        };

        $batch_handler = function ($start_id,$endtime,$maxid,$fLog) use($batch_finder,$batch_checker,$batch_delter)
        {
            $logs = $batch_finder($start_id,$endtime,$maxid);
            foreach ($logs as $batch)
            {
                [$logids,$savers] = $batch_checker($batch);

                $batch_delter($logids);
                foreach ($savers as $item) {
                    fputcsv($fLog,$item);
                }
            }
        };

        ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
        if($start_id === 0) {
            $start_id = $lgid_getter();
        }

        $cur_time = time();
        $sdate = date('Y-m-d',time());
        $filename = BASE_DATA_PATH . "/log/pdlog/{$sdate}-{$cur_time}.csv";
        $fLog = fopen($filename,'w+');

        $logid = $start_id;
        $maxid = $maxid_getter();

//        $onebyone_handler($start_id,$endtime,$maxid,$fLog);
        $batch_handler($start_id,$endtime,$maxid,$fLog);

        $lgid_writter($logid);
    }

    public function testImport()
    {
        $filename = BASE_DATA_PATH . "/log/pdlog/2022-01-02-1641055749.csv";
        $this->import_pdlog($filename);
    }

    //docker-compose run -d phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::testImportNine)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function testImportNine()
    {
        $filename = BASE_DATA_PATH . "/log/pdlog/2021-09-01.csv";
        $this->import_pdlog($filename);
    }

    //docker-compose run -d phpcli php /var/www/html/phpunit-9.2.5.phar --filter "/(TestRefillClear::testImportTen)( .*)?$/" --test-suffix TestRefillClear.php /var/www/html/test
    public function testImportTen()
    {
        $filename = BASE_DATA_PATH . "/log/pdlog/2022-01-03-1641141137.csv";
        $this->import_pdlog($filename);
    }

    private function import_pdlog($filename,$start_line = 0)
    {
        $position = function ($fp,$line)
        {
            $index = 0;
            while(!feof($fp) && $index < $line) {
                $items = fgetcsv($fp);
                $index += 1;
            }
        };

        $reader = function ($fp)
        {
            while(!feof($fp))
            {
                $logs = [];
                for ($index = 0; $index < 200 && !feof($fp); $index++) {
                    $log = fgetcsv($fp);
                    $logs[] = $log;
                }

                yield $logs;
            }
        };

        $converter = function ($logs)
        {
            $result = [];
            $sns = [];
            foreach ($logs as $log) {
                $item = [];

                $logid = $log[0];
                $item['lg_id'] = $log[0];
                $item['lg_member_id'] = $log[1];
                $item['lg_member_name'] = $log[2];
                $item['lg_admin_name'] = $log[3];
                $item['lg_type'] = $log[4];
                $item['lg_av_amount'] = $log[5];
                $item['lg_freeze_amount'] = $log[6];
                $item['lg_add_time'] = $log[7];
                $item['lg_desc'] = $log[8];
                $item['lg_order_sn'] = $log[9];
                $item['lg_available'] = $log[10];

                $sns[] = $log[9];
                $result[$logid] = $item;
            }

            return [$result,$sns];
        };

        $mod_refill = Model('refill_order');
        $filter  = function ($logs,$sns) use($mod_refill)
        {
            $sns = array_unique($sns);

            $sn2sn = [];
            $orders = $mod_refill->table('refill_order')->field('order_sn')->where(['order_sn' => ['in', $sns]])->master(true)->select();
            foreach ($orders as $order) {
                $sn = $order['order_sn'];
                $sn2sn[$sn] = $sn;
            }

            $inserts = [];
            $dels = [];
            foreach ($logs as $lgid => $log)
            {
                $sn = $log['lg_order_sn'];
                if(array_key_exists($sn,$sn2sn)) {
                    $inserts[] = $log;
                } else {
                    $dels[] = $log;
                }
            }

            return [$inserts,$dels];
        };

        $inserter = function ($records)
        {
            if(empty($records)) return false;

            try {
                $ret = Model()->table(self::pdlog_table_name)->insertAll($records);
                return $ret;
            }
            catch (Exception $ex) {
                Log::record($ex->getMessage(),Log::ERR);
                return false;
            }
        };

        $delname = BASE_DATA_PATH . "/log/pdlog/del.csv";
        $fDel = fopen($delname,'a+');
        $delter = function ($records) use ($fDel)
        {
            foreach ($records as $item) {
                $log = [];
                $log[] = $item['lg_id'];
                $log[] = $item['lg_member_id'];
                $log[] = $item['lg_member_name'];
                $log[] = $item['lg_admin_name'];
                $log[] = $item['lg_type'];
                $log[] = $item['lg_av_amount'];
                $log[] = $item['lg_freeze_amount'];
                $log[] = $item['lg_add_time'];
                $log[] = $item['lg_desc'];
                $log[] = $item['lg_order_sn'];
                $log[] = $item['lg_available'];

                fputcsv($fDel,$log);
            }
        };

        ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
        $fLog = fopen($filename,'r');
        $position($fLog,$start_line);

        $logger = $reader($fLog);
        foreach ($logger as $items) {
            [$logs, $sns] = $converter($items);
            [$inserts, $dels] = $filter($logs, $sns);
            $inserter($inserts);
            $delter($dels);
        }

        fclose($fLog);
        fclose($fDel);
        ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    }
}