Skip to content

Instantly share code, notes, and snippets.

@GroxExMachine

GroxExMachine/KeywordForecastManagerV2.php

Last active March 29, 2019 12:51
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save GroxExMachine/a077bb1da5701e8c165eaadbee367954 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save GroxExMachine/a077bb1da5701e8c165eaadbee367954 to your computer and use it in GitHub Desktop.
Download ZIP
Yandex Direct Forecast Manager (2017)
Raw
KeywordForecastManagerV2.php
<?php
namespace AppBundle\Yandex\Keyword;
use AppBundle\Entity\YandexBannerPhraseInfo;
use AppBundle\Entity\YandexErrorLog;
use AppBundle\Entity\YandexKeywordForecast;
use AppBundle\Entity\YandexKeywordForecastRequest;
use AppBundle\Entity\YandexPhraseAuctionBid;
use Doctrine\ORM\EntityManager;
use AppBundle\Provider\YandexDirect;
use AppBundle\Service\RequestLogger;
use AppBundle\Yandex\DirectApiV4;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\ProgressBar;
// TODO Works in progress, refactoring
class KeywordForecastManagerV2 extends DirectApiV4
{
const TIME_BETWEEN_REQUESTS_IN_SECONDS = 15;
const TIME_BETWEEN_REQUESTS_IN_MICROSECONDS = self::TIME_BETWEEN_REQUESTS_IN_SECONDS * 1000000;
const CURRENCY = 'RUB';
const MAX_PHRASES_IN_FORECAST = 100;
const MAX_NUMBER_OF_FORECASTS = 5;
const FORECAST_STATUSES = [
'New' => 0,
'Done' => 1,
'Pending' => 2,
'Failed' => 3
];
const PROGRESS_BAR_SIZE = 30;
const PROGRESS_BAR_FORMAT
= "\n%message% \n"
. "%current%/%max% [%bar%] %percent:3s%% \n"
. "Time: %elapsed:6s%/%estimated:-6s% %memory:6s% \n"
//. "%forecast_1% %forecast_1_status% %forecast_1_time%\n"
. "%forecast_1% %forecast_1_status% \n"
. "%forecast_2% %forecast_2_status% \n"
. "%forecast_3% %forecast_3_status% \n"
. "%forecast_4% %forecast_4_status% \n"
. "%forecast_5% %forecast_5_status% \n"
. "Action log: %action_log% \n"
. "Action: %action% \n"
;
const PROGRESS_BAR_MESSAGES = [
'action' => 'action',
'action_log' => 'action_log',
'forecast_1' => 'forecast_1',
'forecast_1_status' => 'forecast_1_status',
'forecast_2' => 'forecast_2',
'forecast_2_status' => 'forecast_2_status',
'forecast_3' => 'forecast_3',
'forecast_3_status' => 'forecast_3_status',
'forecast_4' => 'forecast_4',
'forecast_4_status' => 'forecast_4_status',
'forecast_5' => 'forecast_5',
'forecast_5_status' => 'forecast_5_status',
];
const REGIONS = [
'Москва и область' => 1,
'Орловская область' => 10772,
'Смоленская область' => 10795,
'Санкт-Петербург и Ленинградская область' => 10174,
'Мурманская область' => 10897,
'Нижегородская область' => 11079,
'Пермский край' => 11108,
'Республика Башкортостан' => 11111,
'Казань' => 43,
'Самарская область' => 11131,
'Краснодарский край' => 10995,
'Иркутская область' => 11266,
'Красноярский край' => 11309,
'Новосибирская область' => 11316,
'Омская область' => 11318,
'Приморский край' => 11409,
'Республика Саха (Якутия)' => 11443,
'Хабаровский край' => 11457,
'Свердловская область' => 11162,
'Ханты-Мансийский автономный округ' => 11193
];
// work in progress
const EVENTS = [
'EVENT_DELETE_BEFORE' => 1,
'EVENT_DELETE_AFTER' => 2
];
const EVENT_DELETE_BEFORE = 1;
const EVENT_DELETE_AFTER = 2;
private $regions;
private $entityManager;
private $requestLogger;
/**
* @var OutputInterface
*/
private $output;
/**
* @var ProgressBar
*/
private $progress;
private $phrases;
/**
* @var array
*/
private $forecastRequests;
public function __construct(EntityManager $entityManager, RequestLogger $requestLogger)
{
$this->entityManager = $entityManager;
$this->requestLogger = $requestLogger;
$this->requestLogger->setApiVersion(4);
$this->regions = array_values(self::REGIONS);
}
public function setOutput(OutputInterface $output)
{
$this->output = $output;
}
private function logger($string = '', $sameLine = false)
{
static $log = [];
$string = date('H:i:s') . ' ' . $string;
//$string = $sameLine ? "\r$string" : "\n$string";
//echo $string;
// $this->progress->setMessage(implode('; ', $log), self::PROGRESS_BAR_MESSAGES['action_log']);
if ($sameLine) {
array_pop($log);
}
$log[] = $string;
if (count($log) > 4) {
array_shift($log);
}
$this->setActionMsg($string);
return $string;
}
private function getTimer()
{
static $time;
// remember previous time
$lastTime = $time;
// start new timer
$time = time();
if ($lastTime) {
return time() - $lastTime;
}
return 0;
}
public function commandLineInterface()
{
$this->batchForecast();
}
/**
* Creating, updating status, getting, waiting, while there are no more forecasts on Yandex
*/
public function batchForecast()
{
$this->prepareProgressBar();
$this->progress->start($this->getNumberOfForecastsToRequest());
exit;
$startTime = time();
while (true) {
// reset timer
$this->getTimer();
// todo add how many keywords left
// todo add estimation numbers
// check server side and update statuses locally
$this->manageServerList();
// get all forecasts that done
$savedForecasts = $this->getForecasts();
if (count($savedForecasts)) {
// delete from Yandex
$this->delete($savedForecasts);
}
// create forecasts
$this->batchCreateForecastForGroup();
$this->entityManager->flush();
$this->entityManager->clear();
// exit condition
if ($this->isStopBatchForecast()) {
break;
}
$timeDifference = self::TIME_BETWEEN_REQUESTS_IN_SECONDS - $this->getTimer();
$timeout = $timeDifference > 0 ? $timeDifference : 0;
for ($i = $timeout; $i > 0; $i--) {
$msg = 'Wait for ' . $i . 's ';
$this->logger($msg, true);
sleep(1);
}
}
$timePassed = round((time() - $startTime) / 60, 1);
$this->logger();
$msg = 'Done. Time passed: ' . $timePassed . ' min';
$this->logger($msg);
}
/**
* Get the list of Forecasts from Yandex
* @return array
*/
public function getForecastList()
{
$msg = 'GetForecastList';
$this->logger($msg);
$service = $this->getService(self::SERVICE_GET);
$forecastList = $service->GetForecastList();
if ($forecastList === false) {
$this->errorHandler($service->getLastError());
return false;
}
$msg = 'Number of forecasts: ' . count($forecastList);
$this->logger($msg);
return $forecastList;
}
/**
* @return bool|int
*/
public function createForecastForGroup()
{
$Keywords = $this->entityManager->getRepository('AppBundle:Keyword')
->findBy([
'Deleted' => false,
'ForecastRequestId' => null,
'YandexErrorLog' => null
], null, self::MAX_PHRASES_IN_FORECAST);
if (!$Keywords) {
$msg = 'There are no available Keywords.';
$this->logger($msg);
return false;
}
$this->phrases = [];
foreach ($Keywords as $Keyword) {
$this->phrases[$Keyword->getId()] = $Keyword->getPhrase();
}
$forecastId = $this->createForecast();
if (!$forecastId) {
$msg = 'Create Forecast: Error.';
$this->logger($msg);
return false;
}
$this->progressListAdd($forecastId);
// $msg = 'id: ' . $forecastId;
// $this->logger($msg);
foreach ($Keywords as $Keyword) {
$Keyword->setForecastRequestId($forecastId);
}
$this->entityManager->flush();
return $forecastId;
}
/**
* @return bool|int
*/
public function createYandexForecast()
{
$msg = 'Create Yandex Forecast';
$this->logger($msg);
$service = $this->getService(self::SERVICE_CREATE);
$NewForecastInfo = new YandexDirect\ApiV4Live\StructType\NewForecastInfo();
$NewForecastInfo
->setPhrases($this->phrases)
->setCurrency(self::CURRENCY)
->setAuctionBids('Yes')
->setGeoID($this->regions);
$forecastId = $service->CreateNewForecast($NewForecastInfo);
if (!$forecastId) {
$this->errorHandler($service->getLastError());
return false;
}
return $forecastId;
}
/**
* Service to get data from Yandex for Forecasts that is ready (status=Done).
* @return array
*/
public function getForecasts()
{
$msg = 'Get Forecasts';
$this->logger($msg);
$YandexKeywordForecastRequests = $this->entityManager->getRepository('AppBundle:YandexKeywordForecastRequest')
->findBy([
'StatusForecast' => self::FORECAST_STATUSES['Done'],
'Received' => null
]);
if (count($YandexKeywordForecastRequests) === 0) {
$msg = 'There are no ready forecasts in DB';
$this->logger($msg);
return [];
}
$service = $this->getService(self::SERVICE_GET);
$savedForecasts = [];
foreach ($YandexKeywordForecastRequests as $request) {
$yandexId = $request->getYandexId();
$msg = 'Get id: ' . $yandexId;
$this->logger($msg);
$this->progressListUpdateItem($yandexId, 'Get');
$forecast = $service->GetForecast($yandexId);
if (!$forecast) {
$msg = 'Error for id: ' . $request->getId() . '. YandexId: ' . $yandexId;
$this->logger($msg);
$this->errorHandler($service->getLastError());
continue;
}
$YandexKeywordForecastId = $this->saveForecastResults($request, $forecast);
$savedForecasts[] = $YandexKeywordForecastId;
$this->progressListUpdateItem($YandexKeywordForecastId, 'Saved');
// update forecast request, mark it processed
$request->setReceived(new \DateTime());
$this->progress->advance();
}
$this->entityManager->flush();
return $savedForecasts;
}
private function isStopBatchForecast()
{
$numberOfForecastsOnServer = $this->manageServerList();
if ($numberOfForecastsOnServer) {
return false;
}
$Keywords = $this->entityManager->getRepository('AppBundle:Keyword')
->findBy([
'Deleted' => false,
'ForecastRequestId' => null,
'YandexErrorLog' => null
]);
if ($Keywords && count($Keywords)) {
return false;
}
return true;
}
/**
* Get the lists of Forecasts from Yandex
* Update statuses locally
* Delete forecast that we do not need
* @return bool
*/
private function manageServerList()
{
// GET
$forecastList = $this->getForecastList();
if ($forecastList === false) {
return false;
}
// UPDATE
$results = $this->updateStatus($forecastList);
// REMOVE
if (count($results['notFound']) !== 0) {
$this->delete($results['notFound']);
}
$this->entityManager->flush();
return count($results['updated']);
}
/**
* @param \AppBundle\Provider\YandexDirect\ApiV4Live\StructType\GetForecastInfo[] $forecasts
* @return array
*/
private function updateStatus(array $forecasts)
{
$results = [
'updated' => [],
'notFound' => []
];
foreach ($forecasts as $forecast) {
$msg = 'Forecast id: ' . $forecast->ForecastID . ', status: ' . $forecast->StatusForecast . '. ';
$this->logger($msg);
//$this->updateForecastRequestStatus($forecast->ForecastID, $forecast->StatusForecast);
$YandexKeywordForecastRequest = $this->entityManager->getRepository('AppBundle:YandexKeywordForecastRequest')
->findOneBy([
'YandexId' => $forecast->ForecastID
]);
if ($YandexKeywordForecastRequest) {
$status = static::FORECAST_STATUSES[$forecast->StatusForecast];
$YandexKeywordForecastRequest->setStatusForecast($status);
$results['updated'][] = $forecast->ForecastID;
$this->progressListUpdateItem($forecast->ForecastID, $status);
}
else {
$results['notFound'][] = $forecast->ForecastID;
$msg = 'Local forecast ' . $forecast->ForecastID . ' not found.';
$this->logger($msg);
}
}
return $results;
}
/**
* @param array $ids
* @return int
*/
private function delete(array $ids)
{
$msg = 'Delete';
$this->logger($msg);
$service = $this->getService(self::SERVICE_DELETE);
$results = [];
foreach ($ids as $id) {
$results[$id] = $service->DeleteForecastReport($id);
}
$deleted = 0;
foreach ($ids as $id) {
if ($results[$id] === 1) {
$deleted++;
$this->progressListRemove($id);
$msg = "Forecast $id deleted.";
}
else {
$msg = print_r($service->getLastError(), true);
}
$this->logger($msg);
}
return $deleted;
}
private function batchCreateForecastForGroup()
{
$forecastList = $this->getForecastList();
$forecastsOnServer = count($forecastList);
$limit = self::MAX_NUMBER_OF_FORECASTS - $forecastsOnServer;
if ($limit > 0) {
$msg = "Create $limit forecasts";
$this->logger($msg);
while ($limit > 0) {
$result = $this->createForecastForGroup();
if (!$result) {
break;
}
$limit--;
}
}
}
/**
* @return bool|int
*/
private function createForecast()
{
$msg = 'Create Forecast for: ' . count($this->phrases) . ' phrases';
$this->logger($msg);
$yandexForecastId = $this->createYandexForecast();
if ($yandexForecastId) {
$YandexKeywordForecastRequest = new YandexKeywordForecastRequest();
$YandexKeywordForecastRequest
->setPhrases(implode(', ', $this->phrases))
->setAuctionBids('Yes')
->setYandexId($yandexForecastId);
$this->entityManager->persist($YandexKeywordForecastRequest);
$this->entityManager->flush($YandexKeywordForecastRequest);
return $YandexKeywordForecastRequest->getId();
}
else {
return false;
}
}
/**
* @param \SoapFault[] $errors
*/
private function errorHandler($errors)
{
$properties = [
'faultstring',
'faultcode',
'detail',
'details'
];
foreach ($errors as $error) {
$msg = 'Error: ';
$this->logger($msg);
$msg = ' Message: ' . $error->getMessage();
$this->logger($msg);
$msg = ' Code: ' . $error->getCode();
$this->logger($msg);
$logMsg = '';
foreach ($properties as $property) {
if ($property === 'detail' || $property === 'details' ) {
$strings = explode(';', $error->$property);
foreach ($strings as $string) {
$msg = "$property: " . $string;
$this->logger($msg);
$logMsg .= $msg . '. ';
if ($error->faultstring === 'Invalid phrase') {
$this->markErrorKeyword($string);
}
}
}
elseif(isset($error->$property)) {
$msg = "$property: " . $error->$property;
$this->logger($msg);
$logMsg .= $msg . '. ';
}
}
$this->requestLogger->log('4', 'errorHandler', '', $logMsg);
}
}
private function markErrorKeyword($errorMessage)
{
$result = preg_match('~\"(.*)\"~', $errorMessage, $match);
if (!$result) {
return false;
}
$errorPhrase = $match[1];
foreach ($this->phrases as $id => $phrase) {
if ($errorPhrase === $phrase) {
$YandexErrorLog = new YandexErrorLog();
$YandexErrorLog->setErrorMessage($errorMessage);
$Keyword = $this->entityManager->getRepository('AppBundle:Keyword')->find($id);
$Keyword->setYandexErrorLog($YandexErrorLog);
$this->entityManager->persist($YandexErrorLog);
$this->entityManager->flush($YandexErrorLog);
$this->entityManager->flush($Keyword);
}
}
}
/**
* @param \AppBundle\Entity\YandexKeywordForecastRequest $request
* @param \AppBundle\Provider\YandexDirect\ApiV4Live\StructType\GetForecastInfo $forecast
* @return int
*/
private function saveForecastResults($request, $forecast)
{
$Common = $forecast->getCommon();
$YandexKeywordForecast = new YandexKeywordForecast();
$YandexKeywordForecast->setYandexId($request->getYandexId());
$this->copyProperties($Common, $YandexKeywordForecast, ['YandexBannerPhraseInfo']);
$Phrases = $forecast->getPhrases();
foreach ($Phrases as $Phrase) {
$YandexBannerPhraseInfo = new YandexBannerPhraseInfo();
$this->copyProperties($Phrase, $YandexBannerPhraseInfo, ['AuctionBids']);
$AuctionBids = $Phrase->AuctionBids;
foreach ($AuctionBids as $AuctionBid) {
$YandexPhraseAuctionBid = new YandexPhraseAuctionBid();
$this->copyProperties($AuctionBid, $YandexPhraseAuctionBid);
$YandexBannerPhraseInfo->addYandexPhraseAuctionBid($YandexPhraseAuctionBid);
$YandexPhraseAuctionBid->setYandexBannerPhraseInfo($YandexBannerPhraseInfo);
}
$YandexKeywordForecast->addYandexBannerPhraseInfo($YandexBannerPhraseInfo);
$YandexBannerPhraseInfo->setYandexKeywordForecast($YandexKeywordForecast);
}
$this->entityManager->persist($YandexKeywordForecast);
return $YandexKeywordForecast->getYandexId();
}
/** Copy all properties trough get-set methods
* @param $source
* @param $target
* @param array $exclusions
* @return $target
*/
private function copyProperties($source, $target, array $exclusions = [])
{
$vars = get_object_vars($source);
foreach ($vars as $key => $val) {
$setMethod = 'set' . ucfirst($key);
//echo "Var: $key, Method: $setMethod(".($source->$key).") \n";
if (method_exists($target, $setMethod) && !array_search($key, $exclusions)) {
$target->$setMethod($source->$key);
}
}
return $target;
}
private function getNumberOfKeywordsToProceed()
{
$QB = $this->entityManager->createQueryBuilder();
$numberOfKeywordsToProceed =
$QB
->select($QB->expr()->count('k.id'))
->from('AppBundle:Keyword', 'k')
->where('k.Deleted = false')
->andWhere($QB->expr()->isNull('k.ForecastRequestId'))
->andWhere($QB->expr()->isNull('k.YandexErrorLog'))
->getQuery()
->getSingleScalarResult();
return $numberOfKeywordsToProceed;
}
private function getNumberOfForecastsToRequest()
{
return ceil($this->getNumberOfKeywordsToProceed()/self::MAX_PHRASES_IN_FORECAST);
}
private function prepareProgressBar()
{
$this->progress = new ProgressBar($this->output);
//$this->progress->setBarWidth(self::PROGRESS_BAR_SIZE);
$this->progress->setFormat(self::PROGRESS_BAR_FORMAT);
foreach (explode("\n", self::PROGRESS_BAR_FORMAT) as $item) {
echo "\n";
}
for ($i = 1; $i <= self::MAX_NUMBER_OF_FORECASTS; $i++) {
$this->progress->setMessage(' ', 'forecast_1');
$this->progress->setMessage(' ', 'forecast_'.$i);
$this->progress->setMessage(' ', 'forecast_'.$i.'_status');
}
$this->progress->setMessage(' ');
$this->progress->setMessage(' ', self::PROGRESS_BAR_MESSAGES['action']);
$this->progress->setMessage(' ', self::PROGRESS_BAR_MESSAGES['action_log']);
}
private function progressListAdd($id)
{
$this->progressListUpdateItem($id, self::FORECAST_STATUSES['New']);
}
private function progressListRemove($id)
{
unset($this->forecastRequests[$id]);
}
private function progressListUpdateItem($id, $status)
{
$this->forecastRequests[$id] = $status;
$this->progressUpdateList();
}
private function progressUpdateList()
{
$i = 0;
foreach ($this->forecastRequests as $id => $status) {
$i++;
$this->progress->setMessage($id, 'forecast_'.$i);
$this->progress->setMessage($status, 'forecast_'.$i.'_status');
}
}
private function setActionMsg($msg)
{
$this->progress->setMessage($msg, self::PROGRESS_BAR_MESSAGES['action']);
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment