saas: refactor the way we manage cron jobs

.env

@@ -49,6 +49,7 @@ BASE_URL=localhost:3008/ # my-site.com
 # IF issue with this test account, check security param in https://myaccount.google.com/u/2/security?rapt=AEjHL4MASG0pbZiaYzD2g01fD9vdZFtSVOo3dh_KFqD25qzhflkPkseEwFvYaJj-88us7MWu8-adroSUYBrdQnKnqz91ylTcuA
 MAILER_URL=gmail://test.gogocarto:creerdescartesagogo@localhost
 FROM_EMAIL=test.gogocarto@gmail.com
+MAX_EMAIL_PER_HOUR=70
 ###< symfony/swiftmailer-bundle ###
 
 ###> hwi/oauth-bundle ###

CHANGELOG.md

+v3.2.1
+======
+* MAJOR: The way of managing SAAS instance cron jobs have changed. In your main database run `db.Project.updateMany({}, {$currentDate: {nextUpdateAt: true}})` and check `docs/installation.md` for new cron jobs configuration
+
 v3.1.4
 ======
 * FEATURE: Ability to configure a custom domain in SASS mode (if the server uses nginx)

docs/installation.md

@@ -31,9 +31,8 @@ Main requirements are :
 1. PHP 7.4
 2. [Composer](https://getcomposer.org/download/)
 3. [Nodejs](https://nodejs.org/en/download/)
-4. [Git](https://git-scm.com/)
+4. [MongoDB](http://php.net/manual/fr/mongodb.installation.php)
 5. Web Server (Apache, Nginx)
-6. [MongoDB](http://php.net/manual/fr/mongodb.installation.php)
 
 Please refer to the dockerfile to know all dependencies : [DockerFile](../docker/server/Dockerfile)
 
@@ -86,16 +85,24 @@ Here are the following cron tab you need to configure
 ```shell
 # for a Normal instance
 @daily php GOGOCARTO_DIR/bin/console --env=prod app:elements:checkvote
-@hourly php GOGOCARTO_DIR/bin/console --env=prod app:users:sendNewsletter
-@hourly php GOGOCARTO_DIR/bin/console --env=prod app:webhooks:post
 @daily php GOGOCARTO_DIR/bin/console --env=prod app:elements:checkExternalSourceToUpdate
+@daily php GOGOCARTO_DIR/bin/console --env=prod app:notify-moderation
+
+@hourly php GOGOCARTO_DIR/bin/console --env=prod app:users:sendNewsletter
+*/5 * * * * php GOGOCARTO_DIR/bin/console --env=prod app:webhooks:post
 ```
 
 ```shell
 # for a SAAS instance
-* * * * * php GOGOCARTO_DIR/bin/console --env=prod app:main-command
-@daily php GOGOCARTO_DIR/bin/console --env=prod app:saas:update-projects-info
+* * * * * php GOGOCARTO_DIR/bin/console --env=prod app:project:update
+# If you have more than 1400 project, you should run it twice a minute :
+* * * * * sleep 30 && php GOGOCARTO_DIR/bin/console --env=prod app:project:update
+# Task ran for every projects that need it at once
+@hourly php GOGOCARTO_DIR/bin/console --env=prod app:users:sendNewsletter
+*/5 * * * * php GOGOCARTO_DIR/bin/console --env=prod app:webhooks:post
+
 @daily php GOGOCARTO_DIR/bin/console --env=prod app:projects:check-for-deleting
+# Next one is for custom domain, it works only with NGINX
 * * * * * cd GOGOCARTO_DIR && sh bin/execute_custom_domain.sh
 ```
 
@@ -108,13 +115,3 @@ make gogo-update
 ```
 
 You can have a look to [the CHANGELOG](../CHANGELOG.md) to know what are the new features and breaking changes.
-
-
-Issues
---------------
-
-If memory limits while using Composer:
-
-```shell
-COMPOSER_MEMORY_LIMIT=-1 composer ...
-```

mongo_operations

@@ -149,3 +149,5 @@ db.Element.updateMany({'nonDuplicates.$id': "KBL"}, { $unset: { 'nonDuplicates':
 db.Element.dropIndex('name_text')
 db.Element.dropIndex('search_index')
 db.Element.createIndex( { name: "text", "data.description": "text" }, { name: "search_index", default_language: "french", weights: { name: 10, "data.description": 5 }, })
+
+db.Project.updateMany({}, {$currentDate: {nextUpdateAt: true}})
\ No newline at end of file

src/Command/CheckExternalSourceToUpdateCommand.php

@@ -35,19 +35,20 @@ class CheckExternalSourceToUpdateCommand extends GoGoAbstractCommand
         $dynamicImports = $qb->field('refreshFrequencyInDays')->gt(0)
                 ->field('nextRefresh')->lte(new \DateTime())
                 ->execute();
-
-        $this->log('CheckExternalSourceToUpdate : Nombre de sources à mettre à jour : '.$dynamicImports->count());
-
-        foreach ($dynamicImports as $key => $import) {
-            $this->log('Updating source : '.$import->getSourceName());
-            try {
-                $this->log($this->importService->startImport($import));
-            } catch (\Exception $e) {
-                $this->dm->persist($import);
-                $import->setCurrState(ImportState::Failed);
-                $message = $e->getMessage().'</br>'.$e->getFile().' LINE '.$e->getLine();
-                $import->setCurrMessage($message);
-                $this->error('Source: '.$import->getSourceName().' - '.$message);
+        if ($count = $dynamicImports->count() > 0) {
+            $this->log("CheckExternalSourceToUpdate : Nombre de sources à mettre à jour : $count");
+
+            foreach ($dynamicImports as $import) {
+                $this->log('Updating source : '.$import->getSourceName());
+                try {
+                    $this->log($this->importService->startImport($import));
+                } catch (\Exception $e) {
+                    $this->dm->persist($import);
+                    $import->setCurrState(ImportState::Failed);
+                    $message = $e->getMessage().'</br>'.$e->getFile().' LINE '.$e->getLine();
+                    $import->setCurrMessage($message);
+                    $this->error('Source: '.$import->getSourceName().' - '.$message);
+                }
             }
         }
     }

src/Command/CheckVoteCommand.php

@@ -33,19 +33,15 @@ class CheckVoteCommand extends GoGoAbstractCommand
 
     protected function gogoExecute(DocumentManager $dm, InputInterface $input, OutputInterface $output): void
     {
-        $elementRepo = $dm->get('Element');
-        $elements = $elementRepo->findPendings();
-
-        foreach ($elements as $element) {
-            $this->voteService->checkVotes($element);
-            $dm->persist($element);
+        $elements = $dm->get('Element')->findPendings();
+
+        if (count($elements)) {
+            foreach ($elements as $element) {
+                $this->voteService->checkVotes($element);
+                $dm->persist($element);
+            }
+            $dm->flush();            
+            $this->log('Check Vote, nombre elements checkés : '.count($elements));
         }
-
-        $dm->flush();
-
-        $output->writeln('Nombre elements checkés : '.count($elements));
-
-        // send notif here so we don't need to create another command
-        $this->notifService->sendModerationNotifications();
     }
 }

src/Command/GoGoAbstractCommand.php

@@ -40,20 +40,31 @@ class GoGoAbstractCommand extends Command
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        $this->dm = $this->dmFactory->getCurrentManager();
         try {
             $this->output = $output;
 
+             // create dummy user, as some code called from command will maybe need the current user informations
+             $token = new AnonymousToken('admin', 'GoGo Gadget au Bot', ['ROLE_ADMIN']);
+             $this->security->setToken($token);
+
             if ($input->getArgument('dbname')) {
                 $this->dm = $this->dmFactory->switchCurrManagerToUseDb($input->getArgument('dbname'));
+                $this->gogoExecute($this->dm, $input, $output);
+            } else if ($_ENV['USE_AS_SAAS']) {
+                $qb = $this->dm->query('Project');
+                $this->filterProjects($qb);                        
+                $dbs = $qb->select('domainName')->getArray();
+                $count = count($dbs);                
+                $this->log("---- Run {$this->getName()} for $count projects", false);
+                foreach($dbs as $dbName) {
+                    $this->dm = $this->dmFactory->createForDB($dbName);;
+                    $this->gogoExecute($this->dm, $input, $output);
+                }                
             } else {
                 $this->dm = $this->dmFactory->getRootManager();
+                $this->gogoExecute($this->dm, $input, $output);
             }
-
-            // create dummy user, as some code called from command will maybe need the current user informations
-            $token = new AnonymousToken('admin', 'GoGo Gadget au Bot', ['ROLE_ADMIN']);
-            $this->security->setToken($token);
-
-            $this->gogoExecute($this->dm, $input, $output);
         } catch (\Exception $e) {
             $message = $e->getMessage().'</br>'.$e->getFile().' LINE '.$e->getLine();
             $this->error('Error executing command: '.$message);
@@ -68,14 +79,22 @@ class GoGoAbstractCommand extends Command
     {
     }
 
-    protected function log($message)
+    // when calling the command with dbName, we run it for all projects
+    // Here we cna filter the project that really need to be processed
+    protected function filterProjects($qb)
+    {
+    }
+
+    protected function log($message, $usePrefix = true)
     {
+        if ($usePrefix) $message = "DB {$this->dm->getConfiguration()->getDefaultDB()} : $message";
         $this->logger->info($message);
         $this->output->writeln($message);
     }
 
     protected function error($message)
     {
+        $message = "DB {$this->dm->getConfiguration()->getDefaultDB()} : $message";
         $this->logger->error($message);
         $this->output->writeln('ERROR '.$message);
         $log = new GoGoLog(GoGoLogLevel::Error, 'Error running '.$this->getName().' : '.$message);

src/Command/GoGoMainCommand.php

-<?php
-
-namespace App\Command;
-
-use Doctrine\ODM\MongoDB\DocumentManager;
-use Psr\Log\LoggerInterface;
-use Symfony\Component\Console\Command\Command;
-use Symfony\Component\Console\Input\ArrayInput;
-use Symfony\Component\Console\Input\InputInterface;
-use Symfony\Component\Console\Output\OutputInterface;
-
-/*
-* For SAAS Instance, this command is executed every minute, and check if there is a command to execute
-* for a particular instance. This permit to not run all the commands as the same time
-*/
-class GoGoMainCommand extends Command
-{
-    // List of the command to execute periodically, with the period in hours
-    public const SCHEDULED_COMMANDS = [
-      'app:elements:checkvote' => '24H',
-      'app:elements:checkExternalSourceToUpdate' => '24H',
-      'app:users:sendNewsletter' => '1H',
-      'app:webhooks:post' => '5M', // 5 minuutes
-   ];
-
-    public function __construct(DocumentManager $dm, LoggerInterface $commandsLogger)
-    {
-        $this->dm = $dm;
-        $this->logger = $commandsLogger;
-        parent::__construct();
-    }
-
-    protected function configure()
-    {
-        $this->setName('app:main-command');
-    }
-
-    protected function execute(InputInterface $input, OutputInterface $output)
-    {
-        $qb = $this->dm->query('ScheduledCommand');
-
-        $commandToExecute = $qb->field('nextExecutionAt')->lte(new \DateTime())
-                             ->sort('nextExecutionAt', 'ASC')
-                             ->getQuery()->getSingleResult();
-
-        if (null !== $commandToExecute) {
-            // Updating next execution time
-            $dateNow = new \DateTime();
-            $dateNow->setTimestamp(time());
-            $interval = new \DateInterval('PT'.self::SCHEDULED_COMMANDS[$commandToExecute->getCommandName()]);
-            $commandToExecute->setNextExecutionAt($dateNow->add($interval));
-            $this->dm->persist($commandToExecute);
-            $this->dm->flush();
-
-            try {
-                $this->logger->info('---- Running command '.$commandToExecute->getCommandName().' for project : '.$commandToExecute->getProject()->getName());
-            } catch (\Exception $e) {
-                // the project has been deleted
-                $this->logger->info('---- DELETEING command '.$commandToExecute->getCommandName());
-                $this->dm->remove($commandToExecute);
-                $this->dm->flush();
-
-                return;
-            }
-            $command = $this->getApplication()->find($commandToExecute->getCommandNAme());
-
-            $arguments = [
-               'command' => $commandToExecute->getCommandName(),
-               'dbname' => $commandToExecute->getProject()->getDbName(),
-            ];
-
-            $input = new ArrayInput($arguments);
-            try {
-                $command->run($input, $output);
-            } catch (\Exception $e) {
-                $this->logger->error($e->getMessage());
-            }
-        }
-    }
-}

src/Command/ImportSourceCommand.php

@@ -40,16 +40,14 @@ class ImportSourceCommand extends GoGoAbstractCommand
             $sourceNameOrId = $input->getArgument('sourceNameOrImportId');
             $import = $dm->get('Import')->find($sourceNameOrId);
             if (!$import) {
-                $import = $dm->get('Import')->findOneBySourceName($sourceNameOrId);
+                $import = $dm->get('Import')->findOneBy(['sourceName' => $sourceNameOrId]);
             }
             if (!$import) {
-                $message = "ERREUR pendant l'import : Aucune source avec pour nom ou id ".$input->getArgument('sourceNameOrImportId')." n'existe dans la base de donnée ".$input->getArgument('dbname');
+                $message = "ERREUR pendant l'import : Aucune source avec pour nom ou id ".$input->getArgument('sourceNameOrImportId');
                 $this->error($message);
-
                 return;
             }
-
-            $this->log('Updating source '.$import->getSourceName().' for project '.$input->getArgument('dbname').' begins...');
+            $this->log("Updating source $import->getSourceName() begins...");
             $result = $this->importService->startImport($import, $manuallyStarted = false);
             $this->log($result);
         } catch (\Exception $e) {
@@ -57,7 +55,7 @@ class ImportSourceCommand extends GoGoAbstractCommand
             $import->setCurrState(ImportState::Failed);
             $message = $e->getMessage().'</br>'.$e->getFile().' LINE '.$e->getLine();
             $import->setCurrMessage($message);
-            $this->error('Source: '.$import->getSourceName().' - '.$message);
+            $this->error("Source: $import->getSourceName() - $message");
             $this->notifService->notifyImportError($import);
         }
     }

src/Command/NewsletterCommand.php

@@ -2,10 +2,9 @@
 
 namespace App\Command;
 
-use App\Document\User;
-use App\Services\NewsletterService;
 use Doctrine\ODM\MongoDB\DocumentManager;
 use App\Services\DocumentManagerFactory;
+use App\Services\MailService;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -13,14 +12,12 @@ use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInt
 
 final class NewsletterCommand extends GoGoAbstractCommand
 {
-    private $newsletterService;
-
     public function __construct(DocumentManagerFactory $dm, LoggerInterface $commandsLogger,
-                               TokenStorageInterface $security,
-                               NewsletterService $newsletterService)
+                               TokenStorageInterface $security, MailService $mailService)
     {
-        $this->newsletterService = $newsletterService;
         parent::__construct($dm, $commandsLogger, $security);
+        $this->mailService = $mailService;
+        $this->remainingEmailsCount = $_ENV['MAX_EMAIL_PER_HOUR'];
     }
 
     protected function gogoConfigure(): void
@@ -31,20 +28,39 @@ final class NewsletterCommand extends GoGoAbstractCommand
        ;
     }
 
+    protected function filterProjects($qb)
+    {
+        return $qb->field('haveNewsletter')->equals(true);
+    }
+
+    private $remainingEmailsCount = 0;
+
     protected function gogoExecute(DocumentManager $dm, InputInterface $input, OutputInterface $output): void
     {
-        $usersRepo = $dm->getRepository(User::class);
+        if ($this->remainingEmailsCount <= 0) return; 
+
+        $users = $dm->get('User')->findNeedsToReceiveNewsletter($this->remainingEmailsCount);
+        $usersCount = $users->count();
+        $this->remainingEmailsCount -= $usersCount;
 
-        $users = $usersRepo->findNeedsToReceiveNewsletter();
-        $nbrUsers = $users->count();
+        if ($usersCount)
+            $this->log("Send Newsletter for $usersCount users");
 
-        foreach ($users as $key => $user) {
+        foreach ($users as $user) {
             $dm->persist($user);
-            $nreElements = $this->newsletterService->sendTo($user);
-            // $this->log('  -> User : ' . $user->getDisplayName() . ', location : ' . $user->getLocation() . ' / ' . $user->getNewsletterRange() . ' km -> Nre Elements : ' .  $nreElements);
-        }
+            $elements = $dm->get('Element')->findWithinCenterFromDate(
+                $user->getGeo()->getLatitude(),
+                $user->getGeo()->getLongitude(),
+                $user->getNewsletterRange(),
+                $user->getLastNewsletterSentAt());
 
+            $elementCount = $elements->count();
+            if ($elementCount > 0) {
+                $this->mailService->sendAutomatedMail('newsletter', $user, null, $elements);
+            }
+            $user->setLastNewsletterSentAt(new \DateTime());
+            $user->updateNextNewsletterDate();
+        }
         $dm->flush();
-        $this->log('Nombre newsletters envoyées : '.$nbrUsers);
     }
 }

src/Command/NotifyModerationCommand.php

+<?php
+
+namespace App\Command;
+
+use Doctrine\ODM\MongoDB\DocumentManager;
+use App\Services\DocumentManagerFactory;
+use App\Services\UserNotificationService;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
+
+class NotifyModerationCommand extends GoGoAbstractCommand
+{
+    public function __construct(DocumentManagerFactory $dm, LoggerInterface $commandsLogger,
+                               TokenStorageInterface $security,
+                               UserNotificationService $notifService)
+    {
+        $this->notifService = $notifService;
+        parent::__construct($dm, $commandsLogger, $security);
+    }
+
+    protected function gogoConfigure(): void
+    {
+        $this
+        ->setName('app:notify-moderation')
+        ->setDescription('Notify users about pending moeration or problem on an import');
+    }
+
+    protected function gogoExecute(DocumentManager $dm, InputInterface $input, OutputInterface $output): void
+    {
+        $userNotified = $this->notifService->sendModerationNotifications();
+        if ($userNotified) {
+            $this->log("Notify Moderation, $userNotified users notified");
+        }        
+    }
+}

src/Command/ProjectUpdateCommand.php

+<?php
+
+namespace App\Command;
+
+use App\Services\AsyncService;
+use App\Services\DocumentManagerFactory;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Input\InputArgument;
+
+/*
+* For SAAS Instance, this command run the daily commands
+* one time every day for each project
+*/
+class ProjectUpdateCommand extends Command
+{
+    // List of the command to execute periodically
+    public const DAILY_COMMANDS = [
+      'app:elements:checkvote',
+      'app:elements:checkExternalSourceToUpdate',
+      'app:notify-moderation'
+    ];
+
+    public function __construct(DocumentManagerFactory $dmFactory, LoggerInterface $commandsLogger,
+                                AsyncService $asyncService)
+    {
+        $this->dmFactory = $dmFactory;
+        $this->asyncService = $asyncService;
+        $this->logger = $commandsLogger;
+        parent::__construct();
+    }
+
+    protected function configure()
+    {
+        $this->setName('app:project:update');
+        $this->addArgument('dbname', InputArgument::OPTIONAL, 'Db name');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        try {
+            $rootDm = $this->dmFactory->getRootManager();
+
+            // If project is specified, we update it, otherwise we get get project that need an update
+            if ($input->getArgument('dbname')) {
+                $project = $rootDm->get('Project')->findOneBy(['domainName' => $input->getArgument('dbname')]);
+            } else {
+                $project = $rootDm->query('Project')->field('nextUpdateAt')->lte(new \DateTime())
+                                  ->sort('nextUpdateAt', 'ASC')
+                                  ->getOne();
+            }
+
+            if ($project !== null) {
+                // Updating next execution time
+                $dateNow = new \DateTime();
+                $dateNow->setTimestamp(time());
+                $interval = new \DateInterval('PT24H');
+                $project->setNextUpdateAt($dateNow->add($interval));
+                // Update Project Info - return false means the project is wrongly configured, like without config
+                if (!$this->updateProjectInfo($project)) {
+                    return;
+                }
+
+                $rootDm->persist($project);
+                $rootDm->flush();
+
+                $this->logger->info("---- PROJECT {$project->getDbName()} : Update infos & run daily commands");
+
+                // run daily commands
+                $this->asyncService->setRunSynchronously(true);
+                foreach(self::DAILY_COMMANDS as $commandName) {
+                    $this->asyncService->callCommand($commandName, [], $project->getDbName());
+                }
+            } else {
+                $this->logger->error("PROJECT is null");
+            }
+        } catch (\Exception $e) {
+            $this->logger->error($e->getMessage());
+        }
+    }
+
+    private function updateProjectInfo($project)
+    {
+        $dm = $this->dmFactory->createForDB($project->getDbName());
+
+        $config = $dm->get('Configuration')->findConfiguration();
+        if (!$config) {
+            $this->logger->error("Project {$project->getDomainName()} does not have config");
+            return false;
+        }
+        $img = $config->getSocialShareImage() ? $config->getSocialShareImage() : $config->getLogo();
+        $imageUrl = $img ? $img->getImageUrl() : null;
+        $dataSize = $dm->get('Element')->findVisibles(true);
+
+        $users = $dm->get('User')->findAll();
+        $adminEmails = [];
+        $lastLogin = null;
+        foreach ($users as $key => $user) {
+            if ($user->isAdmin()) $adminEmails[] = $user->getEmail();
+            if (!$lastLogin || $user->getLastLogin() > $lastLogin) $lastLogin = $user->getLastLogin();
+        }
+        $haveWebhooks = $dm->query('Webhook')->count()->execute() > 0
+                     || $dm->query('Import')->field('isSynchronized')->equals(true)->getCount() > 0;
+        $haveNewsletter = $config->getNewsletterMail()->getActive()
+                       && $dm->query('User')->field('newsletterFrequency')->gt(0)->getCount() > 0;
+        
+        $project->setName($config->getAppName());
+        $project->setImageUrl($imageUrl);
+        $project->setDescription($config->getAppBaseline());
+        $project->setDataSize($dataSize);
+        $project->setAdminEmails(implode(',', $adminEmails));
+        $project->setPublished($config->getPublishOnSaasPage());
+        if ($lastLogin) $project->setLastLogin($lastLogin);
+        $project->setHaveWebhooks($haveWebhooks);
+        $project->setHaveNewsletter($haveNewsletter);
+        return true;
+    }
+}

src/Command/UpdateElementsJsonCommand.php

@@ -42,7 +42,7 @@ class UpdateElementsJsonCommand extends GoGoAbstractCommand
 
             $count = $elements->count();
 
-            $this->log('DB Generating json representation for '.$count.' elements...');
+            $this->log('Generating json representation for '.$count.' elements...');
 
             $i = 0;
             foreach ($elements as $key => $element) {

src/Command/UpdateProjectsInfoCommand.php

-<?php
-
-namespace App\Command;
-
-use Doctrine\ODM\MongoDB\DocumentManager;
-use Psr\Log\LoggerInterface;
-use Symfony\Component\Console\Command\Command;
-use Symfony\Component\Console\Input\InputInterface;
-use Symfony\Component\Console\Output\OutputInterface;
-use Symfony\Component\Routing\RouterInterface;
-use App\Services\DocumentManagerFactory;
-
-/*
-* Update infos of each instance for the Saas Index page
-*/
-class UpdateProjectsInfoCommand extends Command
-{
-    public function __construct(DocumentManagerFactory $dmFactory, LoggerInterface $commandsLogger,
-                                RouterInterface $router)
-    {
-        $this->dmFactory = $dmFactory;
-        $this->rootDm = $dmFactory->getRootManager();
-        $this->router = $router;
-        $this->logger = $commandsLogger;
-        parent::__construct();
-    }
-
-    protected function configure()
-    {
-        $this->setName('app:saas:update-projects-info');
-    }
-
-    protected function execute(InputInterface $input, OutputInterface $output)
-    {