Unverified Commit 87aecd04 authored by Sebastian Castro's avatar Sebastian Castro
Browse files

saas: refactor the way we manage cron jobs

parent 55694819
......@@ -49,6 +49,7 @@ BASE_URL=localhost:3008/ # my-site.com
# IF issue with this test account, check security param in https://myaccount.google.com/u/2/security?rapt=AEjHL4MASG0pbZiaYzD2g01fD9vdZFtSVOo3dh_KFqD25qzhflkPkseEwFvYaJj-88us7MWu8-adroSUYBrdQnKnqz91ylTcuA
MAILER_URL=gmail://test.gogocarto:creerdescartesagogo@localhost
FROM_EMAIL=test.gogocarto@gmail.com
MAX_EMAIL_PER_HOUR=70
###< symfony/swiftmailer-bundle ###
###> hwi/oauth-bundle ###
......
v3.2.1
======
* MAJOR: The way of managing SAAS instance cron jobs have changed. In your main database run `db.Project.updateMany({}, {$currentDate: {nextUpdateAt: true}})` and check `docs/installation.md` for new cron jobs configuration
v3.1.4
======
* FEATURE: Ability to configure a custom domain in SASS mode (if the server uses nginx)
......
......@@ -31,9 +31,8 @@ Main requirements are :
1. PHP 7.4
2. [Composer](https://getcomposer.org/download/)
3. [Nodejs](https://nodejs.org/en/download/)
4. [Git](https://git-scm.com/)
4. [MongoDB](http://php.net/manual/fr/mongodb.installation.php)
5. Web Server (Apache, Nginx)
6. [MongoDB](http://php.net/manual/fr/mongodb.installation.php)
Please refer to the dockerfile to know all dependencies : [DockerFile](../docker/server/Dockerfile)
......@@ -86,16 +85,24 @@ Here are the following cron tab you need to configure
```shell
# for a Normal instance
@daily php GOGOCARTO_DIR/bin/console --env=prod app:elements:checkvote
@hourly php GOGOCARTO_DIR/bin/console --env=prod app:users:sendNewsletter
@hourly php GOGOCARTO_DIR/bin/console --env=prod app:webhooks:post
@daily php GOGOCARTO_DIR/bin/console --env=prod app:elements:checkExternalSourceToUpdate
@daily php GOGOCARTO_DIR/bin/console --env=prod app:notify-moderation
@hourly php GOGOCARTO_DIR/bin/console --env=prod app:users:sendNewsletter
*/5 * * * * php GOGOCARTO_DIR/bin/console --env=prod app:webhooks:post
```
```shell
# for a SAAS instance
* * * * * php GOGOCARTO_DIR/bin/console --env=prod app:main-command
@daily php GOGOCARTO_DIR/bin/console --env=prod app:saas:update-projects-info
* * * * * php GOGOCARTO_DIR/bin/console --env=prod app:project:update
# If you have more than 1400 project, you should run it twice a minute :
* * * * * sleep 30 && php GOGOCARTO_DIR/bin/console --env=prod app:project:update
# Task ran for every projects that need it at once
@hourly php GOGOCARTO_DIR/bin/console --env=prod app:users:sendNewsletter
*/5 * * * * php GOGOCARTO_DIR/bin/console --env=prod app:webhooks:post
@daily php GOGOCARTO_DIR/bin/console --env=prod app:projects:check-for-deleting
# Next one is for custom domain, it works only with NGINX
* * * * * cd GOGOCARTO_DIR && sh bin/execute_custom_domain.sh
```
......@@ -108,13 +115,3 @@ make gogo-update
```
You can have a look to [the CHANGELOG](../CHANGELOG.md) to know what are the new features and breaking changes.
Issues
--------------
If memory limits while using Composer:
```shell
COMPOSER_MEMORY_LIMIT=-1 composer ...
```
......@@ -149,3 +149,5 @@ db.Element.updateMany({'nonDuplicates.$id': "KBL"}, { $unset: { 'nonDuplicates':
db.Element.dropIndex('name_text')
db.Element.dropIndex('search_index')
db.Element.createIndex( { name: "text", "data.description": "text" }, { name: "search_index", default_language: "french", weights: { name: 10, "data.description": 5 }, })
db.Project.updateMany({}, {$currentDate: {nextUpdateAt: true}})
\ No newline at end of file
......@@ -35,19 +35,20 @@ class CheckExternalSourceToUpdateCommand extends GoGoAbstractCommand
$dynamicImports = $qb->field('refreshFrequencyInDays')->gt(0)
->field('nextRefresh')->lte(new \DateTime())
->execute();
$this->log('CheckExternalSourceToUpdate : Nombre de sources à mettre à jour : '.$dynamicImports->count());
foreach ($dynamicImports as $key => $import) {
$this->log('Updating source : '.$import->getSourceName());
try {
$this->log($this->importService->startImport($import));
} catch (\Exception $e) {
$this->dm->persist($import);
$import->setCurrState(ImportState::Failed);
$message = $e->getMessage().'</br>'.$e->getFile().' LINE '.$e->getLine();
$import->setCurrMessage($message);
$this->error('Source: '.$import->getSourceName().' - '.$message);
if ($count = $dynamicImports->count() > 0) {
$this->log("CheckExternalSourceToUpdate : Nombre de sources à mettre à jour : $count");
foreach ($dynamicImports as $import) {
$this->log('Updating source : '.$import->getSourceName());
try {
$this->log($this->importService->startImport($import));
} catch (\Exception $e) {
$this->dm->persist($import);
$import->setCurrState(ImportState::Failed);
$message = $e->getMessage().'</br>'.$e->getFile().' LINE '.$e->getLine();
$import->setCurrMessage($message);
$this->error('Source: '.$import->getSourceName().' - '.$message);
}
}
}
}
......
......@@ -33,19 +33,15 @@ class CheckVoteCommand extends GoGoAbstractCommand
protected function gogoExecute(DocumentManager $dm, InputInterface $input, OutputInterface $output): void
{
$elementRepo = $dm->get('Element');
$elements = $elementRepo->findPendings();
foreach ($elements as $element) {
$this->voteService->checkVotes($element);
$dm->persist($element);
$elements = $dm->get('Element')->findPendings();
if (count($elements)) {
foreach ($elements as $element) {
$this->voteService->checkVotes($element);
$dm->persist($element);
}
$dm->flush();
$this->log('Check Vote, nombre elements checkés : '.count($elements));
}
$dm->flush();
$output->writeln('Nombre elements checkés : '.count($elements));
// send notif here so we don't need to create another command
$this->notifService->sendModerationNotifications();
}
}
......@@ -40,20 +40,31 @@ class GoGoAbstractCommand extends Command
protected function execute(InputInterface $input, OutputInterface $output)
{
$this->dm = $this->dmFactory->getCurrentManager();
try {
$this->output = $output;
// create dummy user, as some code called from command will maybe need the current user informations
$token = new AnonymousToken('admin', 'GoGo Gadget au Bot', ['ROLE_ADMIN']);
$this->security->setToken($token);
if ($input->getArgument('dbname')) {
$this->dm = $this->dmFactory->switchCurrManagerToUseDb($input->getArgument('dbname'));
$this->gogoExecute($this->dm, $input, $output);
} else if ($_ENV['USE_AS_SAAS']) {
$qb = $this->dm->query('Project');
$this->filterProjects($qb);
$dbs = $qb->select('domainName')->getArray();
$count = count($dbs);
$this->log("---- Run {$this->getName()} for $count projects", false);
foreach($dbs as $dbName) {
$this->dm = $this->dmFactory->createForDB($dbName);;
$this->gogoExecute($this->dm, $input, $output);
}
} else {
$this->dm = $this->dmFactory->getRootManager();
$this->gogoExecute($this->dm, $input, $output);
}
// create dummy user, as some code called from command will maybe need the current user informations
$token = new AnonymousToken('admin', 'GoGo Gadget au Bot', ['ROLE_ADMIN']);
$this->security->setToken($token);
$this->gogoExecute($this->dm, $input, $output);
} catch (\Exception $e) {
$message = $e->getMessage().'</br>'.$e->getFile().' LINE '.$e->getLine();
$this->error('Error executing command: '.$message);
......@@ -68,14 +79,22 @@ class GoGoAbstractCommand extends Command
{
}
protected function log($message)
// when calling the command with dbName, we run it for all projects
// Here we cna filter the project that really need to be processed
protected function filterProjects($qb)
{
}
protected function log($message, $usePrefix = true)
{
if ($usePrefix) $message = "DB {$this->dm->getConfiguration()->getDefaultDB()} : $message";
$this->logger->info($message);
$this->output->writeln($message);
}
protected function error($message)
{
$message = "DB {$this->dm->getConfiguration()->getDefaultDB()} : $message";
$this->logger->error($message);
$this->output->writeln('ERROR '.$message);
$log = new GoGoLog(GoGoLogLevel::Error, 'Error running '.$this->getName().' : '.$message);
......
<?php
namespace App\Command;
use Doctrine\ODM\MongoDB\DocumentManager;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\ArrayInput;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
/*
* For SAAS Instance, this command is executed every minute, and check if there is a command to execute
* for a particular instance. This permit to not run all the commands as the same time
*/
class GoGoMainCommand extends Command
{
// List of the command to execute periodically, with the period in hours
public const SCHEDULED_COMMANDS = [
'app:elements:checkvote' => '24H',
'app:elements:checkExternalSourceToUpdate' => '24H',
'app:users:sendNewsletter' => '1H',
'app:webhooks:post' => '5M', // 5 minuutes
];
public function __construct(DocumentManager $dm, LoggerInterface $commandsLogger)
{
$this->dm = $dm;
$this->logger = $commandsLogger;
parent::__construct();
}
protected function configure()
{
$this->setName('app:main-command');
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$qb = $this->dm->query('ScheduledCommand');
$commandToExecute = $qb->field('nextExecutionAt')->lte(new \DateTime())
->sort('nextExecutionAt', 'ASC')
->getQuery()->getSingleResult();
if (null !== $commandToExecute) {
// Updating next execution time
$dateNow = new \DateTime();
$dateNow->setTimestamp(time());
$interval = new \DateInterval('PT'.self::SCHEDULED_COMMANDS[$commandToExecute->getCommandName()]);
$commandToExecute->setNextExecutionAt($dateNow->add($interval));
$this->dm->persist($commandToExecute);
$this->dm->flush();
try {
$this->logger->info('---- Running command '.$commandToExecute->getCommandName().' for project : '.$commandToExecute->getProject()->getName());
} catch (\Exception $e) {
// the project has been deleted
$this->logger->info('---- DELETEING command '.$commandToExecute->getCommandName());
$this->dm->remove($commandToExecute);
$this->dm->flush();
return;
}
$command = $this->getApplication()->find($commandToExecute->getCommandNAme());
$arguments = [
'command' => $commandToExecute->getCommandName(),
'dbname' => $commandToExecute->getProject()->getDbName(),
];
$input = new ArrayInput($arguments);
try {
$command->run($input, $output);
} catch (\Exception $e) {
$this->logger->error($e->getMessage());
}
}
}
}
......@@ -40,16 +40,14 @@ class ImportSourceCommand extends GoGoAbstractCommand
$sourceNameOrId = $input->getArgument('sourceNameOrImportId');
$import = $dm->get('Import')->find($sourceNameOrId);
if (!$import) {
$import = $dm->get('Import')->findOneBySourceName($sourceNameOrId);
$import = $dm->get('Import')->findOneBy(['sourceName' => $sourceNameOrId]);
}
if (!$import) {
$message = "ERREUR pendant l'import : Aucune source avec pour nom ou id ".$input->getArgument('sourceNameOrImportId')." n'existe dans la base de donnée ".$input->getArgument('dbname');
$message = "ERREUR pendant l'import : Aucune source avec pour nom ou id ".$input->getArgument('sourceNameOrImportId');
$this->error($message);
return;
}
$this->log('Updating source '.$import->getSourceName().' for project '.$input->getArgument('dbname').' begins...');
$this->log("Updating source $import->getSourceName() begins...");
$result = $this->importService->startImport($import, $manuallyStarted = false);
$this->log($result);
} catch (\Exception $e) {
......@@ -57,7 +55,7 @@ class ImportSourceCommand extends GoGoAbstractCommand
$import->setCurrState(ImportState::Failed);
$message = $e->getMessage().'</br>'.$e->getFile().' LINE '.$e->getLine();
$import->setCurrMessage($message);
$this->error('Source: '.$import->getSourceName().' - '.$message);
$this->error("Source: $import->getSourceName() - $message");
$this->notifService->notifyImportError($import);
}
}
......
......@@ -2,10 +2,9 @@
namespace App\Command;
use App\Document\User;
use App\Services\NewsletterService;
use Doctrine\ODM\MongoDB\DocumentManager;
use App\Services\DocumentManagerFactory;
use App\Services\MailService;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
......@@ -13,14 +12,12 @@ use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInt
final class NewsletterCommand extends GoGoAbstractCommand
{
private $newsletterService;
public function __construct(DocumentManagerFactory $dm, LoggerInterface $commandsLogger,
TokenStorageInterface $security,
NewsletterService $newsletterService)
TokenStorageInterface $security, MailService $mailService)
{
$this->newsletterService = $newsletterService;
parent::__construct($dm, $commandsLogger, $security);
$this->mailService = $mailService;
$this->remainingEmailsCount = $_ENV['MAX_EMAIL_PER_HOUR'];
}
protected function gogoConfigure(): void
......@@ -31,20 +28,39 @@ final class NewsletterCommand extends GoGoAbstractCommand
;
}
protected function filterProjects($qb)
{
return $qb->field('haveNewsletter')->equals(true);
}
private $remainingEmailsCount = 0;
protected function gogoExecute(DocumentManager $dm, InputInterface $input, OutputInterface $output): void
{
$usersRepo = $dm->getRepository(User::class);
if ($this->remainingEmailsCount <= 0) return;
$users = $dm->get('User')->findNeedsToReceiveNewsletter($this->remainingEmailsCount);
$usersCount = $users->count();
$this->remainingEmailsCount -= $usersCount;
$users = $usersRepo->findNeedsToReceiveNewsletter();
$nbrUsers = $users->count();
if ($usersCount)
$this->log("Send Newsletter for $usersCount users");
foreach ($users as $key => $user) {
foreach ($users as $user) {
$dm->persist($user);
$nreElements = $this->newsletterService->sendTo($user);
// $this->log(' -> User : ' . $user->getDisplayName() . ', location : ' . $user->getLocation() . ' / ' . $user->getNewsletterRange() . ' km -> Nre Elements : ' . $nreElements);
}
$elements = $dm->get('Element')->findWithinCenterFromDate(
$user->getGeo()->getLatitude(),
$user->getGeo()->getLongitude(),
$user->getNewsletterRange(),
$user->getLastNewsletterSentAt());
$elementCount = $elements->count();
if ($elementCount > 0) {
$this->mailService->sendAutomatedMail('newsletter', $user, null, $elements);
}
$user->setLastNewsletterSentAt(new \DateTime());
$user->updateNextNewsletterDate();
}
$dm->flush();
$this->log('Nombre newsletters envoyées : '.$nbrUsers);
}
}
<?php
namespace App\Command;
use Doctrine\ODM\MongoDB\DocumentManager;
use App\Services\DocumentManagerFactory;
use App\Services\UserNotificationService;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
class NotifyModerationCommand extends GoGoAbstractCommand
{
public function __construct(DocumentManagerFactory $dm, LoggerInterface $commandsLogger,
TokenStorageInterface $security,
UserNotificationService $notifService)
{
$this->notifService = $notifService;
parent::__construct($dm, $commandsLogger, $security);
}
protected function gogoConfigure(): void
{
$this
->setName('app:notify-moderation')
->setDescription('Notify users about pending moeration or problem on an import');
}
protected function gogoExecute(DocumentManager $dm, InputInterface $input, OutputInterface $output): void
{
$userNotified = $this->notifService->sendModerationNotifications();
if ($userNotified) {
$this->log("Notify Moderation, $userNotified users notified");
}
}
}
<?php
namespace App\Command;
use App\Services\AsyncService;
use App\Services\DocumentManagerFactory;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Input\InputArgument;
/*
* For SAAS Instance, this command run the daily commands
* one time every day for each project
*/
class ProjectUpdateCommand extends Command
{
// List of the command to execute periodically
public const DAILY_COMMANDS = [
'app:elements:checkvote',
'app:elements:checkExternalSourceToUpdate',
'app:notify-moderation'
];
public function __construct(DocumentManagerFactory $dmFactory, LoggerInterface $commandsLogger,
AsyncService $asyncService)
{
$this->dmFactory = $dmFactory;
$this->asyncService = $asyncService;
$this->logger = $commandsLogger;
parent::__construct();
}
protected function configure()
{
$this->setName('app:project:update');
$this->addArgument('dbname', InputArgument::OPTIONAL, 'Db name');
}
protected function execute(InputInterface $input, OutputInterface $output)
{
try {
$rootDm = $this->dmFactory->getRootManager();
// If project is specified, we update it, otherwise we get get project that need an update
if ($input->getArgument('dbname')) {
$project = $rootDm->get('Project')->findOneBy(['domainName' => $input->getArgument('dbname')]);
} else {
$project = $rootDm->query('Project')->field('nextUpdateAt')->lte(new \DateTime())
->sort('nextUpdateAt', 'ASC')
->getOne();
}
if ($project !== null) {
// Updating next execution time
$dateNow = new \DateTime();
$dateNow->setTimestamp(time());
$interval = new \DateInterval('PT24H');
$project->setNextUpdateAt($dateNow->add($interval));
// Update Project Info - return false means the project is wrongly configured, like without config
if (!$this->updateProjectInfo($project)) {
return;
}
$rootDm->persist($project);
$rootDm->flush();
$this->logger->info("---- PROJECT {$project->getDbName()} : Update infos & run daily commands");
// run daily commands
$this->asyncService->setRunSynchronously(true);
foreach(self::DAILY_COMMANDS as $commandName) {
$this->asyncService->callCommand($commandName, [], $project->getDbName());
}
} else {
$this->logger->error("PROJECT is null");
}
} catch (\Exception $e) {
$this->logger->error($e->getMessage());
}
}
private function updateProjectInfo($project)
{
$dm = $this->dmFactory->createForDB($project->getDbName());
$config = $dm->get('Configuration')->findConfiguration();
if (!$config) {
$this->logger->error("Project {$project->getDomainName()} does not have config");
return false;
}
$img = $config->getSocialShareImage() ? $config->getSocialShareImage() : $config->getLogo();
$imageUrl = $img ? $img->getImageUrl() : null;
$dataSize = $dm->get('Element')->findVisibles(true);
$users = $dm->get('User')->findAll();
$adminEmails = [];
$lastLogin = null;
foreach ($users as $key => $user) {
if ($user->isAdmin()) $adminEmails[] = $user->getEmail();
if (!$lastLogin || $user->getLastLogin() > $lastLogin) $lastLogin = $user->getLastLogin();
}
$haveWebhooks = $dm->query('Webhook')->count()->execute() > 0
|| $dm->query('Import')->field('isSynchronized')->equals(true)->getCount() > 0;
$haveNewsletter = $config->getNewsletterMail()->getActive()
&& $dm->query('User')->field('newsletterFrequency')->gt(0)->getCount() > 0;
$project->setName($config->getAppName());
$project->setImageUrl($imageUrl);
$project->setDescription($config->getAppBaseline());
$project->setDataSize($dataSize);
$project->setAdminEmails(implode(',', $adminEmails));
$project->setPublished($config->getPublishOnSaasPage());
if ($lastLogin) $project->setLastLogin($lastLogin);
$project->setHaveWebhooks($haveWebhooks);
$project->setHaveNewsletter($haveNewsletter);
return true;
}
}
......@@ -42,7 +42,7 @@ class UpdateElementsJsonCommand extends GoGoAbstractCommand
$count = $elements->count();
$this->log('DB Generating json representation for '.$count.' elements...');
$this->log('Generating json representation for '.$count.' elements...');
$i = 0;
foreach ($elements as $key => $element) {
......
<?php
namespace App\Command;
use Doctrine\ODM\MongoDB\DocumentManager;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Routing\RouterInterface;
use App\Services\DocumentManagerFactory;
/*
* Update infos of each instance for the Saas Index page
*/
class UpdateProjectsInfoCommand extends Command
{
public function __construct(DocumentManagerFactory $dmFactory, LoggerInterface $commandsLogger,
RouterInterface $router)
{
$this->dmFactory = $dmFactory;
$this->rootDm = $dmFactory->getRootManager();
$this->router = $router;
$this->logger = $commandsLogger;
parent::__construct();
}
protected function configure()
{
$this->setName('app:saas:update-projects-info');
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$projects = $this->rootDm->get('Project')->findAll();
$this->logger->info('Updating projects informations. '. count($projects) .' projects to update');