Commit 64a6b5f6 authored by Sebastian Castro's avatar Sebastian Castro

Adds source and externalSource entities. Ability to import Json source, and update it automatically

parent f2b32afa
......@@ -149,8 +149,9 @@ php bin/console doctrine:mongodb:fixtures:load ;
chmod -R 777 var/;
# adding crontab task
line="5 3 * * * php /var/www/html/GoGoCarto/bin/console --env=prod app:elements:checkvote"
line="@daily php /var/www/html/GoGoCarto/bin/console --env=prod app:elements:checkvote"
line2="@hourly php /var/www/html/GoGoCarto/bin/console --env=prod app:users:sendNewsletter"
line3="@daily php /var/www/html/GoGoCarto/bin/console --env=prod app:elements:checkExternalSourceToUpdate"
# TODO add crontab automatically
# (crontab -l; echo "$line" ) | crontab -u userhere -
......
......@@ -13,7 +13,12 @@ class ImportAdmin extends AbstractAdmin
protected function configureFormFields(FormMapper $formMapper)
{
$formMapper
->add('sourceName', 'text', array('required' => false, 'label' => 'Nom de la source'))
->add('source', 'sonata_type_model', array(), array(
'class'=> 'Biopen\GeoDirectoryBundle\Document\Source',
'required' => true,
'choices_as_values' => true,
'label' => 'Source des données',
'mapped' => true))
->add('file', 'file', array('label' => 'Fichier à importer'))
->add('geocodeIfNecessary', null, array('required' => false, 'label' => 'Géocoder si élements sans latitude ni longitude'))
->add('createMissingOptions', null, array('required' => false, 'label' => 'Créer les options manquantes'))
......
<?php
namespace Biopen\GeoDirectoryBundle\Admin;
use Sonata\AdminBundle\Admin\AbstractAdmin;
use Sonata\AdminBundle\Datagrid\ListMapper;
use Sonata\AdminBundle\Datagrid\DatagridMapper;
use Sonata\AdminBundle\Form\FormMapper;
use Sonata\AdminBundle\Route\RouteCollection;
class SourceAdmin extends AbstractAdmin
{
protected function configureFormFields(FormMapper $formMapper)
{
$formMapper
->add('name', null, array('required' => false, 'label' => "Nom de la source"));
}
protected function configureDatagridFilters(DatagridMapper $datagridMapper)
{
$datagridMapper
->add('name')
;
}
protected function configureListFields(ListMapper $listMapper)
{
$listMapper
->addIdentifier('name', null, array('label' => 'Nom de la source'))
->add('_action', 'actions', array(
'actions' => array(
'edit' => array(),
'delete' => array(),
)
))
;
}
}
\ No newline at end of file
<?php
namespace Biopen\GeoDirectoryBundle\Admin;
use Sonata\AdminBundle\Admin\AbstractAdmin;
use Sonata\AdminBundle\Datagrid\ListMapper;
use Sonata\AdminBundle\Datagrid\DatagridMapper;
use Sonata\AdminBundle\Form\FormMapper;
use Sonata\AdminBundle\Route\RouteCollection;
class SourceExternalAdmin extends AbstractAdmin
{
public function getTemplate($name)
{
switch ($name) {
case 'edit': return '@BiopenAdmin/edit/edit_source_external.html.twig';
break;
default : return parent::getTemplate($name);
break;
}
}
protected function configureFormFields(FormMapper $formMapper)
{
$formMapper
->add('name', null, array('required' => false, 'label' => "Nom de la source"))
->add('url', null, array('required' => false, 'label' => "Url de l'api Json"))
->add('refreshFrequencyInDays', null, array('required' => false, 'label' => "Fréquence de mise à jours des données en jours (laisser vide pour ne jamais mettre à jour automatiquement"));
}
protected function configureRoutes(RouteCollection $collection)
{
$collection->add('refresh', $this->getRouterIdParameter().'/refresh');
}
protected function configureDatagridFilters(DatagridMapper $datagridMapper)
{
$datagridMapper
->add('name')
;
}
protected function configureListFields(ListMapper $listMapper)
{
$listMapper
->addIdentifier('name', null, array('label' => 'Nom de la source'))
->add('lastRefresh', 'datetime', array('label' => 'Dernière synchronisation des données', 'format' => 'd/m/Y - H:i'))
->add('nextRefresh', 'date', array('label' => 'Prochaine synchronisation', 'template' => '@BiopenAdmin/partials/list_next_refresh.html.twig'))
->add('_action', 'actions', array(
'actions' => array(
'edit' => array(),
'delete' => array(),
'refresh' => array('template' => '@BiopenAdmin/partials/list__action_refresh.html.twig'),
)
))
;
}
}
\ No newline at end of file
<?php
namespace Biopen\GeoDirectoryBundle\Command;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Output\OutputInterface;
use Biopen\GeoDirectoryBundle\Document\ElementStatus;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
class CheckExternalSourceToUpdateCommand extends ContainerAwareCommand
{
protected function configure()
{
$this
->setName('app:elements:checkExternalSourceToUpdate')
->setDescription('Check for updating external sources');
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$em = $this->getContainer()->get('doctrine_mongodb.odm.default_document_manager');
$qb = $em->createQueryBuilder('BiopenGeoDirectoryBundle:SourceExternal');
$sourcesToUpdate = $qb->field('refreshFrequencyInDays')->gt(0)
->field('nextRefresh')->lte(new \DateTime())
->getQuery()->execute();
$importService = $this->getContainer()->get('biopen.element_import');
$output->writeln('Nombre de sources à mettre à jour : ' . $sourcesToUpdate->count());
foreach ($sourcesToUpdate as $key => $source)
{
$importService->importJson($source);
$output->writeln('Updating source : ' . $source->getName());
}
}
}
\ No newline at end of file
<?php
namespace Biopen\GeoDirectoryBundle\Command;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\ProgressBar;
use Biopen\GeoDirectoryBundle\Document\Element;
use Biopen\GeoDirectoryBundle\Document\ElementStatus;
use Biopen\GeoDirectoryBundle\Document\Coordinates;
use Biopen\GeoDirectoryBundle\Document\OptionValue;
use Acme\AcmeBundle\Entity\User;
class ImportColibrisLmcCommand extends ContainerAwareCommand
{
protected function configure()
{
// Name and description for app/console command
$this
->setName('import:colibris_lmc')
->setDescription('Import colibris and lmc elements from CSV file')
->addArgument('path', InputArgument::REQUIRED, 'The path to csv file')
->addArgument('geocode', InputArgument::REQUIRED, 'Geocoding missing locations or not');
}
protected function execute(InputInterface $input, OutputInterface $output)
{
// Showing when the script is launched
$now = new \DateTime();
$output->writeln('<comment>Start : ' . $now->format('d-m-Y G:i:s') . ' ---</comment>');
// Importing CSV on DB via Doctrine ORM
$this->getContainer()->get('biopen.import_csv')->import($input->getArgument('path'), $input->getArgument('geocode'), $output);
// Showing when the script is over
$now = new \DateTime();
$output->writeln('<comment>End : ' . $now->format('d-m-Y G:i:s') . ' ---</comment>');
}
}
\ No newline at end of file
......@@ -23,7 +23,7 @@ class ImportAdminController extends Controller
private function executeImport($import)
{
$result = $this->get('biopen.import_csv')->import($import);
$result = $this->get('biopen.element_import')->importCsv($import);
if ($result === null)
$this->addFlash('sonata_flash_error', "Un erreur s'est produite lors du chargement du fichier Csv. Vérifiez que le fichier est bien valide");
else
......
<?php
namespace Biopen\GeoDirectoryBundle\Controller\Admin;
use Sonata\AdminBundle\Controller\CRUDController as Controller;
use Symfony\Component\HttpFoundation\RedirectResponse;
use Symfony\Component\HttpFoundation\Request;
class SourceExternalAdminController extends Controller
{
public function refreshAction()
{
$object = $this->admin->getSubject();
$result = $this->get('biopen.element_import')->importJson($object);
if ($result === null)
$this->addFlash('sonata_flash_error', "Un erreur s'est produite lors du chargement du fichier Json. Vérifiez que le fichier est bien valide");
else {
$this->addFlash('sonata_flash_success', 'Les ' . $result .' éléments ont été importés avec succès ');
}
return $this->redirect($this->admin->generateUrl('list'));
}
}
\ No newline at end of file
......@@ -32,12 +32,12 @@ class ImportController extends Controller
return new Response('Elements générés');
}
public function importCsvAction($fileName, $geocode)
{
$this->get('biopen.import_csv')->import($fileName, $geocode);
// public function importCsvAction($fileName, $geocode)
// {
// $this->get('biopen.element_import')->importCsv($fileName, $geocode);
return new Response('Elements importés');
}
// return new Response('Elements importés');
// }
public function availableOptionsAction()
{
......
......@@ -242,6 +242,13 @@ class Element
*/
public $sourceKey = 'PDCN';
/**
* The source from where the element has been imported or created
*
* @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Source")
*/
private $source;
/**
* @var string
*
......@@ -338,8 +345,8 @@ class Element
private $userOwnerEmail;
/**
* When actions are made by many person (like moderation, duplicates check...) we look the elements currently procced by someone
* so noone else make acction on the same element
* When actions are made by many person (like moderation, duplicates check...) we lock the elements currently proceed by someone
* so noone else make action on the same element
* @MongoDB\Field(type="int")
*/
private $lockUntil = 0;
......@@ -1642,4 +1649,26 @@ class Element
{
return $this->lockUntil;
}
/**
* Set source
*
* @param Biopen\GeoDirectoryBundle\Document\Source $source
* @return $this
*/
public function setSource(\Biopen\GeoDirectoryBundle\Document\Source $source)
{
$this->source = $source;
return $this;
}
/**
* Get source
*
* @return Biopen\GeoDirectoryBundle\Document\Source $source
*/
public function getSource()
{
return $this->source;
}
}
......@@ -23,14 +23,14 @@ class Import extends AbstractFile
/**
* @var string
* @MongoDB\Field(type="string")
* @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Source", cascade={"persist"})
*/
private $sourceName = "";
private $source;
/**
* @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Category", cascade={"all"})
* @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Category", cascade={"persist"})
*/
private $parentCategoryToCreateOptions;
private $parentCategoryToCreateOptions = null;
/**
* @MongoDB\EmbedOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Option")
......@@ -40,12 +40,12 @@ class Import extends AbstractFile
/**
* @MongoDB\Field(type="bool")
*/
private $createMissingOptions;
private $createMissingOptions = false;
/**
* @MongoDB\Field(type="bool")
*/
private $geocodeIfNecessary;
private $geocodeIfNecessary = false;
public function __construct() {}
......@@ -59,28 +59,6 @@ class Import extends AbstractFile
return $this->id;
}
/**
* Set sourceName
*
* @param string $sourceName
* @return $this
*/
public function setSourceName($sourceName)
{
$this->sourceName = $sourceName;
return $this;
}
/**
* Get sourceName
*
* @return string $sourceName
*/
public function getSourceName()
{
return $this->sourceName;
}
/**
* Set parentCategoryToCreateOptions
*
......@@ -168,4 +146,26 @@ class Import extends AbstractFile
{
return $this->geocodeIfNecessary;
}
/**
* Set source
*
* @param Biopen\GeoDirectoryBundle\Document\Source $source
* @return $this
*/
public function setSource(\Biopen\GeoDirectoryBundle\Document\Source $source)
{
$this->source = $source;
return $this;
}
/**
* Get source
*
* @return Biopen\GeoDirectoryBundle\Document\Source $source
*/
public function getSource()
{
return $this->source;
}
}
<?php
namespace Biopen\GeoDirectoryBundle\Document;
use Doctrine\ODM\MongoDB\Mapping\Annotations as MongoDB;
/**
* Source
*
* @MongoDB\Document
*/
class Source
{
/**
* @var int
* @MongoDB\Id(strategy="INCREMENT")
*/
protected $id;
/**
* @var string
* @MongoDB\Field(type="string")
*/
protected $name;
public function __toString() { return $this->name; }
/**
* Get id
*
* @return int_id $id
*/
public function getId()
{
return $this->id;
}
/**
* Set name
*
* @param string $name
* @return $this
*/
public function setName($name)
{
$this->name = $name;
return $this;
}
/**
* Get name
*
* @return string $name
*/
public function getName()
{
return $this->name;
}
}
\ No newline at end of file
<?php
namespace Biopen\GeoDirectoryBundle\Document;
use Doctrine\ODM\MongoDB\Mapping\Annotations as MongoDB;
use \Datetime;
/**
* External source to load dynamically
*
* @MongoDB\Document
*/
class SourceExternal extends Source
{
/**
* @var string
* @MongoDB\Field(type="string")
*/
private $url;
/**
* @var string
* @MongoDB\Field(type="int")
*/
private $refreshFrequencyInDays;
/**
* @var date $lastRefresh
*
* @MongoDB\Field(type="date")
*/
private $lastRefresh = null;
/**
* @var date $lastRefresh
*
* @MongoDB\Field(type="date")
*/
private $nextRefresh = null;
public function updateNextRefreshDate()
{
if ($this->getRefreshFrequencyInDays() == 0) $this->setNextRefresh(null);
else
{
$interval = new \DateInterval('P' . $this->getRefreshFrequencyInDays() .'D');
$date = new DateTime();
$date->setTimestamp($this->getLastRefresh());
$this->setNextRefresh($date->add($interval));
}
}
/**
* Set url
*
* @param string $url
* @return $this
*/
public function setUrl($url)
{
$this->url = $url;
return $this;
}
/**
* Get url
*
* @return string $url
*/
public function getUrl()
{
return $this->url;
}
/**
* Set refreshFrequencyInDays
*
* @param int $refreshFrequencyInDays
* @return $this
*/
public function setRefreshFrequencyInDays($refreshFrequencyInDays)
{
$this->refreshFrequencyInDays = $refreshFrequencyInDays;
return $this;
}
/**
* Get refreshFrequencyInDays
*
* @return int $refreshFrequencyInDays
*/
public function getRefreshFrequencyInDays()
{
return $this->refreshFrequencyInDays;
}
/**
* Set lastRefresh
*
* @param date $lastRefresh
* @return $this
*/
public function setLastRefresh($lastRefresh)
{
$this->lastRefresh = $lastRefresh;
return $this;
}
/**
* Get lastRefresh
*
* @return date $lastRefresh
*/
public function getLastRefresh()
{
return $this->lastRefresh;
}
/**
* Set nextRefresh
*
* @param date $nextRefresh
* @return $this
*/
public function setNextRefresh($nextRefresh)
{
$this->nextRefresh = $nextRefresh;
return $this;
}
/**
* Get nextRefresh
*
* @return date $nextRefresh