Refactor Import and Import dynamic

src/Biopen/GeoDirectoryBundle/Admin/ImportAdmin.php

@@ -13,8 +13,8 @@ class ImportAdmin extends AbstractAdmin
     protected function configureFormFields(FormMapper $formMapper)
     {
         $formMapper
-            ->with("Importer des données depuis un fichier CSV", 
-                    ["description" => "Les colonnes importantes du CSV sont les suivantes : 
+            ->with("Importer des données depuis un fichier CSV ou une API Json", 
+                    ["description" => "Les colonnes/propriétés importantes sont les suivantes : 
                     <ul>
                     <li><b>name</b> Le titre de la fiche</li>
                     <li><b>taxonomy</b> la liste des options séparées par des virgules. Exple: Alimentation, Restaurant
@@ -27,19 +27,27 @@ class ImportAdmin extends AbstractAdmin
                     </ul>
                     Vous pouvez ensuite avoir n'importe quelles autres colonnes, elles seront importées. Veillez à faire concorder le nom des colonnes avec le nom des champs de votre formulaire"
                     ])
-                // ->add('source', 'sonata_type_model', array(), array(
-                //     'class'=> 'Biopen\GeoDirectoryBundle\Document\Source', 
-                //     'required' => true, 
-                //     'label' => 'Source des données',
-                //     'mapped' => true))
-                ->add('file', 'file', array('label' => 'Fichier à importer'))
-                ->add('geocodeIfNecessary', null, array('required' => false, 'label' => 'Géocoder si élements sans latitude ni longitude'))
+                ->add('sourceName', 'text', array(
+                    'required' => true,                  
+                    'label' => 'Nom de la source des données'))
+                ->add('file', 'file', array('label' => 'Fichier à importer', 'required' => false))
+                ->add('url', 'text', array('label' => 'Ou Url vers une API Json', 'required' => false))
+                ->add('geocodeIfNecessary', null, array('required' => false, 'label' => 'Géocoder les élements sans latitude ni longitude à partir de leur adresse'))
                 ->add('createMissingOptions', null, array('required' => false, 'label' => 'Créer les options manquantes'))
                 // ->add('parentCategoryToCreateOptions', 'sonata_type_model', array(
                 //     'class'=> 'Biopen\GeoDirectoryBundle\Document\Category', 
                 //     'required' => false, 
+                //     'btn_add' => false,
                 //     'label' => 'Catégorie parente pour créer les options manquantes',
                 //     'mapped' => true), array('admin_code' => 'admin.category'))
+                ->add('createMissingOptions', null, array('required' => false, 'label' => 'Créer les options manquantes à partir des catégories renseignées dans chaque élément'))
+                ->add('optionsToAddToEachElement', 'sonata_type_model', array(
+                    'class'=> 'Biopen\GeoDirectoryBundle\Document\Option', 
+                    'required' => false, 
+                    'choices_as_values' => true,
+                    'multiple' => true,
+                    'btn_add' => false,
+                    'label' => 'Options à ajouter à chaque élément importé'), array('admin_code' => 'admin.option'))                
             ->end()
         ;
     }

src/Biopen/GeoDirectoryBundle/Admin/SourceExternalAdmin.php → src/Biopen/GeoDirectoryBundle/Admin/ImportDynamicAdmin.php

@@ -8,12 +8,12 @@ use Sonata\AdminBundle\Datagrid\DatagridMapper;
 use Sonata\AdminBundle\Form\FormMapper;
 use Sonata\AdminBundle\Route\RouteCollection;
 
-class SourceExternalAdmin extends AbstractAdmin
+class ImportDynamicAdmin extends AbstractAdmin
 {
     public function getTemplate($name) 
     {
         switch ($name) {
-            case 'edit': return '@BiopenAdmin/edit/edit_source_external.html.twig';
+            case 'edit': return '@BiopenAdmin/edit/edit_import_dynamic.html.twig';
             break;
             default : return parent::getTemplate($name);
             break;
@@ -23,8 +23,17 @@ class SourceExternalAdmin extends AbstractAdmin
     protected function configureFormFields(FormMapper $formMapper)
     {
         $formMapper
-            ->add('name', null, array('required' => false, 'label' => "Nom de la source"))
-            ->add('url', null, array('required' => false, 'label' => "Url de l'api Json"))
+            ->add('sourceName', 'text', array('required' => true, 'label' => 'Nom de la source '))
+            ->add('url', 'text', array('label' => "Url de l'api Json", 'required' => true))
+            // ->add('geocodeIfNecessary', null, array('required' => false, 'label' => 'Géocoder les élements sans latitude ni longitude à partir de leur adresse'))
+            ->add('createMissingOptions', null, array('required' => false, 'label' => 'Créer les options manquantes à partir des catégories renseignées dans chaque élément'))
+            ->add('optionsToAddToEachElement', 'sonata_type_model', array(
+                'class'=> 'Biopen\GeoDirectoryBundle\Document\Option', 
+                'required' => false, 
+                'choices_as_values' => true,
+                'multiple' => true,
+                'btn_add' => false,
+                'label' => 'Options à ajouter à chaque élément importé'), array('admin_code' => 'admin.option'))      
             ->add('refreshFrequencyInDays', null, array('required' => false, 'label' => "Fréquence de mise à jours des données en jours (laisser vide pour ne jamais mettre à jour automatiquement"));
     }
 
@@ -36,13 +45,13 @@ class SourceExternalAdmin extends AbstractAdmin
     protected function configureDatagridFilters(DatagridMapper $datagridMapper)
     {
         $datagridMapper
-            ->add('name')
+            ->add('sourceName')
         ;
     }
     protected function configureListFields(ListMapper $listMapper)
     {
         $listMapper
-            ->addIdentifier('name', null, array('label' => 'Nom de la source'))
+            ->addIdentifier('sourceName', null, array('label' => 'Nom de la source'))
             ->add('lastRefresh', 'datetime', array('label' => 'Dernière synchronisation des données', 'format' => 'd/m/Y - H:i'))
             ->add('nextRefresh', 'date', array('label' => 'Prochaine synchronisation', 'template' => '@BiopenAdmin/partials/list_next_refresh.html.twig'))
             ->add('_action', 'actions', array(

src/Biopen/GeoDirectoryBundle/Admin/SourceAdmin.php

-<?php
-
-namespace Biopen\GeoDirectoryBundle\Admin;
-
-use Sonata\AdminBundle\Admin\AbstractAdmin;
-use Sonata\AdminBundle\Datagrid\ListMapper;
-use Sonata\AdminBundle\Datagrid\DatagridMapper;
-use Sonata\AdminBundle\Form\FormMapper;
-use Sonata\AdminBundle\Route\RouteCollection;
-
-class SourceAdmin extends AbstractAdmin
-{
-
-    protected function configureFormFields(FormMapper $formMapper)
-    {
-        $formMapper
-            ->add('name', null, array('required' => false, 'label' => "Nom de la source"));
-    }
-
-    protected function configureDatagridFilters(DatagridMapper $datagridMapper)
-    {
-        $datagridMapper
-            ->add('name')
-        ;
-    }
-    protected function configureListFields(ListMapper $listMapper)
-    {
-        $listMapper
-            ->addIdentifier('name', null, array('label' => 'Nom de la source'))
-            ->add('_action', 'actions', array(
-                'actions' => array(
-                    'edit' => array(),
-                    'delete' => array(),
-                )
-            ))
-        ;
-    }
-}
\ No newline at end of file

src/Biopen/GeoDirectoryBundle/Command/CheckExternalSourceToUpdateCommand.php

@@ -20,19 +20,19 @@ class CheckExternalSourceToUpdateCommand extends GoGoAbstractCommand
     }
 
     protected function gogoExecute($em, InputInterface $input, OutputInterface $output)
-    {
-      $qb = $em->createQueryBuilder('BiopenGeoDirectoryBundle:SourceExternal');
+    {      
+      $qb = $em->createQueryBuilder('BiopenGeoDirectoryBundle:ImportDynamic');
       
-      $sourcesToUpdate = $qb->field('refreshFrequencyInDays')->gt(0)               
+      $dynamicImports = $qb->field('refreshFrequencyInDays')->gt(0)               
                 ->field('nextRefresh')->lte(new \DateTime())
                 ->getQuery()->execute();
       $importService = $this->getContainer()->get('biopen.element_import');
 
-      $this->log('CheckExternalSourceToUpdate : Nombre de sources à mettre à jour : ' . $sourcesToUpdate->count());
+      $this->log('CheckExternalSourceToUpdate : Nombre de sources à mettre à jour : ' . $dynamicImports->count());
 
-      foreach ($sourcesToUpdate as $key => $source)
+      foreach ($dynamicImports as $key => $import)
       { 
-        $this->log('Updating source : ' . $source->getName());
+        $this->log('Updating source : ' . $import->getSource()->getName());
         $dataToImport = $importService->importJson($source);        
       }      
     }

src/Biopen/GeoDirectoryBundle/Command/GenerateElementsCommand.php

@@ -6,11 +6,11 @@ use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Output\OutputInterface;
 
-use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Biopen\SaasBundle\Command\GoGoAbstractCommand;
 
-class GenerateElementsCommand extends ContainerAwareCommand
+class GenerateElementsCommand extends GoGoAbstractCommand
 {
-    protected function configure()
+    protected function gogoConfigure()
     {
        $this
         ->setName('app:elements:generate')
@@ -21,10 +21,12 @@ class GenerateElementsCommand extends ContainerAwareCommand
     ;
     }
 
-    protected function execute(InputInterface $input, OutputInterface $output)
+    protected function gogoExecute($em, InputInterface $input, OutputInterface $output)
     {
+      $this->output = $output;
+
       $this->getContainer()->get('biopen.random_creation_service')->generate($input->getArgument('number'), $input->getArgument('generateVotes'));
 
-      $output->writeln('Element générés !');
+      $this->log('Element générés !');
     }
 }
\ No newline at end of file

src/Biopen/GeoDirectoryBundle/Command/ImportSourceCommand.php

@@ -17,27 +17,29 @@ class ImportSourceCommand extends GoGoAbstractCommand
        $this
         ->setName('app:elements:importSource')
         ->setDescription('Check for updating external sources')
-        ->addArgument('sourceName', InputArgument::REQUIRED, 'The name of the source');
+        ->addArgument('sourceNameOrImportId', InputArgument::REQUIRED, 'The name of the source');
     }
 
     protected function gogoExecute($em, InputInterface $input, OutputInterface $output)
     {
       try {
-        $source = $em->getRepository('BiopenGeoDirectoryBundle:SourceExternal')->findOneByName($input->getArgument('sourceName'));
         $this->output = $output;
+        $sourceNameOrId = $input->getArgument('sourceNameOrImportId');
+        $import = $em->getRepository('BiopenGeoDirectoryBundle:ImportDynamic')->find($sourceNameOrId);
+        if (!$import) $import = $em->getRepository('BiopenGeoDirectoryBundle:ImportDynamic')->findOneBySourceName($sourceNameOrId); 
 
-        if (!$source) 
+        if (!$import) 
         {
-          $message = "ERREUR pendant l'import : Aucune source avec pour nom " . $input->getArgument('sourceName') . " n'existe dans la base de donnée " . $input->getArgument('dbname');
+          $message = "ERREUR pendant l'import : Aucune source avec pour nom ou id " . $input->getArgument('sourceNameOrImportId') . " n'existe dans la base de donnée " . $input->getArgument('dbname');
           $this->error($message);
           return;
         }
-        $this->log('Updating source ' . $source->getName() . ' for project ' . $input->getArgument('dbname') . ' begins...');
+        $this->log('Updating source ' . $import->getSourceName() . ' for project ' . $input->getArgument('dbname') . ' begins...');
         $this->log('Downloading the data...');
         $importService = $this->getContainer()->get('biopen.element_import');
-        $dataToImport = $importService->importJson($source, true);
+        $dataToImport = $importService->importJson($import, true);
         $this->log('Data downloaded. ' . count($dataToImport) . ' elements to import...');  
-        $count = $importService->import($dataToImport, $source);
+        $count = $importService->importData($dataToImport, $import);
         $this->log('Updating source completed : ' . $count . ' elements successfully imported');  
       } catch (\Exception $e) {
           $this->error($e->getMessage());

src/Biopen/GeoDirectoryBundle/Controller/Admin/ImportAdminController.php

@@ -23,13 +23,14 @@ class ImportAdminController extends Controller
 
     private function executeImport($import)
     {        
-        $result = $this->get('biopen.element_import')->importCsv($import);
+        $result = $this->get('biopen.element_import')->startImport($import);
         if ($result === null) 
             $this->addFlash('sonata_flash_error', "Un erreur s'est produite lors du chargement du fichier Csv. Vérifiez que le fichier est bien valide");
         else
             $this->addFlash('sonata_flash_success', 'Les ' . $result .' éléments ont été importés avec succès ');
     }
 
+
     // This method is just an overwrite of the SonataAdminCRUDController for calling the executeImport once the document is created
     public function createAction()
     {
@@ -76,8 +77,7 @@ class ImportAdminController extends Controller
                 $this->admin->checkAccess('create', $object);
 
                 try {
-                    $object = $this->admin->create($object);                    
-
+                    $object = $this->admin->create($object);   
                     $this->executeImport($object);
 
                     // redirect to create mode

src/Biopen/GeoDirectoryBundle/Controller/Admin/SourceExternalAdminController.php → src/Biopen/GeoDirectoryBundle/Controller/Admin/ImportDynamicAdminController.php

@@ -6,13 +6,13 @@ use Sonata\AdminBundle\Controller\CRUDController as Controller;
 use Symfony\Component\HttpFoundation\RedirectResponse;
 use Symfony\Component\HttpFoundation\Request;
 
-class SourceExternalAdminController extends Controller
+class ImportDynamicAdminController extends Controller
 {
     public function refreshAction()
     {
         $object = $this->admin->getSubject();
 
-        $this->get('biopen.async')->callCommand('app:elements:importSource', [$object->getName()]);
+        $this->get('biopen.async')->callCommand('app:elements:importSource', [$object->getId()]);
         $this->addFlash('sonata_flash_success', "Les éléments sont en cours d'importation. Cela peut prendre plusieurs minutes.");
 
         // $dataToImport = $this->get('biopen.element_import')->importJson($object);        

src/Biopen/GeoDirectoryBundle/Document/Element.php

@@ -215,7 +215,7 @@ class Element
     /**
      * The source from where the element has been imported or created
      *
-     * @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Source")
+     * @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Import")
      */
     private $source;
 
@@ -692,6 +692,23 @@ class Element
         $this->privateData = null;
     }
 
+    public function setCustomData($data, $privateProps)
+    {
+        $privateData = [];
+        foreach ($privateProps as $key => $prop) {
+            if (array_key_exists($prop, $data)) {
+                $privateData[$prop] = $data[$prop];
+                unset($data[$prop]);
+            }
+        }
+
+        if ($this->getData()) $data = array_merge($this->getData(), $data); // keeping also old data
+        $this->setData($data);
+        
+        if ($this->getPrivateData()) $privateData = array_merge($this->getPrivateData(), $privateData); // keeping also old data
+        $this->setPrivateData($privateData);
+    }
+
     /**
      * Set status
      *
@@ -1398,10 +1415,10 @@ class Element
     /**
      * Set source
      *
-     * @param Biopen\GeoDirectoryBundle\Document\Source $source
+     * @param Biopen\GeoDirectoryBundle\Document\Import $source
      * @return $this
      */
-    public function setSource(\Biopen\GeoDirectoryBundle\Document\Source $source)
+    public function setSource(\Biopen\GeoDirectoryBundle\Document\Import $source)
     {
         $this->source = $source;
         return $this;
@@ -1410,7 +1427,7 @@ class Element
     /**
      * Get source
      *
-     * @return Biopen\GeoDirectoryBundle\Document\Source $source
+     * @return Biopen\GeoDirectoryBundle\Document\Import $source
      */
     public function getSource()
     {

src/Biopen/GeoDirectoryBundle/Document/Import.php

@@ -10,6 +10,8 @@ use Biopen\CoreBundle\Document\AbstractFile;
 /** 
 * @MongoDB\Document 
 * @Vich\Uploadable
+* Import data into GoGoCarto. the data can imported through a static file, or via API url
+* The Import can be made once for all (static import) or dynamically every X days (ImportDynamic)
 */
 class Import extends AbstractFile
 {
@@ -22,20 +24,27 @@ class Import extends AbstractFile
     private $id;
 
     /**
-     * @var string
-     * @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Source", cascade={"persist"})
+     * @var string   
+     * @MongoDB\Field(type="string")
      */
-    private $source;
+    public $sourceName;
 
     /**
-     * @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Category", cascade={"persist"})
+     * @var string   
+     * Url of API to get the data
+     * @MongoDB\Field(type="string")
+     */
+    private $url;
+
+    /**
+     * @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Category")
      */
     private $parentCategoryToCreateOptions = null;
 
     /**
-     * @MongoDB\EmbedOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Option")
+     * @MongoDB\ReferenceMany(targetDocument="Biopen\GeoDirectoryBundle\Document\Option")
      */
-    private $optionTemplate;
+    private $optionsToAddToEachElement = [];
 
     /**
      * @MongoDB\Field(type="bool")
@@ -49,6 +58,8 @@ class Import extends AbstractFile
     
     public function __construct() {}
 
+    public function isDynamicImport() { return false; }
+
     /**
      * Get id
      *
@@ -82,26 +93,26 @@ class Import extends AbstractFile
     }
 
     /**
-     * Set optionTemplate
+     * Set url
      *
-     * @param Biopen\GeoDirectoryBundle\Document\Option $optionTemplate
+     * @param string $url
      * @return $this
      */
-    public function setOptionTemplate(\Biopen\GeoDirectoryBundle\Document\Option $optionTemplate)
+    public function setUrl($url)
     {
-        $this->optionTemplate = $optionTemplate;
+        $this->url = $url;
         return $this;
     }
 
     /**
-     * Get optionTemplate
+     * Get url
      *
-     * @return Biopen\GeoDirectoryBundle\Document\Option $optionTemplate
+     * @return string $url
      */
-    public function getOptionTemplate()
+    public function getUrl()
     {
-        return $this->optionTemplate;
-    }
+        return $this->url;
+    }    
 
     /**
      * Set createMissingOptions
@@ -148,24 +159,54 @@ class Import extends AbstractFile
     }
 
     /**
-     * Set source
+     * Add optionsToAddToEachElement
+     *
+     * @param Biopen\GeoDirectoryBundle\Document\Option $optionsToAddToEachElement
+     */
+    public function addOptionsToAddToEachElement(\Biopen\GeoDirectoryBundle\Document\Option $optionsToAddToEachElement)
+    {
+        $this->optionsToAddToEachElement[] = $optionsToAddToEachElement;
+    }
+
+    /**
+     * Remove optionsToAddToEachElement
+     *
+     * @param Biopen\GeoDirectoryBundle\Document\Option $optionsToAddToEachElement
+     */
+    public function removeOptionsToAddToEachElement(\Biopen\GeoDirectoryBundle\Document\Option $optionsToAddToEachElement)
+    {
+        $this->optionsToAddToEachElement->removeElement($optionsToAddToEachElement);
+    }
+
+    /**
+     * Get optionsToAddToEachElement
+     *
+     * @return \Doctrine\Common\Collections\Collection $optionsToAddToEachElement
+     */
+    public function getOptionsToAddToEachElement()
+    {
+        return $this->optionsToAddToEachElement;
+    }
+
+    /**
+     * Set sourceName
      *
-     * @param Biopen\GeoDirectoryBundle\Document\Source $source
+     * @param string $sourceName
      * @return $this
      */
-    public function setSource(\Biopen\GeoDirectoryBundle\Document\Source $source)
+    public function setSourceName($sourceName)
     {
-        $this->source = $source;
+        $this->sourceName = $sourceName;
         return $this;
     }
 
     /**
-     * Get source
+     * Get sourceName
      *
-     * @return Biopen\GeoDirectoryBundle\Document\Source $source
+     * @return string $sourceName
      */
-    public function getSource()
+    public function getSourceName()
     {
-        return $this->source;
+        return $this->sourceName;
     }
 }

src/Biopen/GeoDirectoryBundle/Document/SourceExternal.php → src/Biopen/GeoDirectoryBundle/Document/ImportDynamic.php

@@ -10,14 +10,8 @@ use \Datetime;
  *
  * @MongoDB\Document
  */
-class SourceExternal extends Source
+class ImportDynamic extends Import
 {
-    /**
-     * @var string   
-     * @MongoDB\Field(type="string")
-     */
-    private $url;
-
     /**
      * @var string   
      * @MongoDB\Field(type="int")
@@ -38,7 +32,7 @@ class SourceExternal extends Source
      */
     private $nextRefresh = null;
 
-    public function isExternalsource() { return true; }
+    public function isDynamicImport() { return true; }
 
     public function updateNextRefreshDate() 
     {
@@ -52,28 +46,6 @@ class SourceExternal extends Source
         }
     }    
 
-    /**
-     * Set url
-     *
-     * @param string $url
-     * @return $this
-     */
-    public function setUrl($url)
-    {
-        $this->url = $url;
-        return $this;
-    }
-
-    /**
-     * Get url
-     *
-     * @return string $url
-     */
-    public function getUrl()
-    {
-        return $this->url;
-    }    
-
     /**
      * Set refreshFrequencyInDays
      *

src/Biopen/GeoDirectoryBundle/Document/Option.php

@@ -188,7 +188,7 @@ class Option
 
     public function __toString() 
     {
-        return "(Option) " . $this->getName();
+        return $this->getName();
     }
 
     public function getNameWithParent()

src/Biopen/GeoDirectoryBundle/Document/Source.php

-<?php
-
-namespace Biopen\GeoDirectoryBundle\Document;
-
-use Doctrine\ODM\MongoDB\Mapping\Annotations as MongoDB;
-
-/**
- * Source
- *
- * @MongoDB\Document
- */
-class Source
-{
-    /**
-     * @var int
-     * @MongoDB\Id(strategy="INCREMENT") 
-     */
-    protected $id;
-
-    /**
-     * @var string   
-     * @MongoDB\Field(type="string")
-     */
-    protected $name;
-
-    public function __toString() { return $this->name; }
-
-    public function isExternalsource() { return false; }
-    /**
-     * Get id
-     *
-     * @return int_id $id
-     */
-    public function getId()
-    {
-        return $this->id;
-    }
-
-    /**
-     * Set name
-     *
-     * @param string $name
-     * @return $this
-     */
-    public function setName($name)
-    {
-        $this->name = $name;
-        return $this;
-    }
-
-    /**
-     * Get name
-     *
-     * @return string $name
-     */
-    public function getName()
-    {
-        return $this->name;
-    }
-}
\ No newline at end of file

src/Biopen/GeoDirectoryBundle/Resources/config/services_admin.yml

@@ -42,8 +42,8 @@ services:
             - { name: sonata.admin, manager_type: doctrine_mongodb, group: "data", label: Import } 
 
     admin.source_external:
-        class: Biopen\GeoDirectoryBundle\Admin\SourceExternalAdmin
-        arguments: [~, Biopen\GeoDirectoryBundle\Document\SourceExternal, 'BiopenGeoDirectoryBundle:Admin/SourceExternalAdmin' ]
+        class: Biopen\GeoDirectoryBundle\Admin\ImportDynamicAdmin
+        arguments: [~, Biopen\GeoDirectoryBundle\Document\ImportDynamic, 'BiopenGeoDirectoryBundle:Admin/ImportDynamicAdmin' ]
         tags:
             - { name: sonata.admin, manager_type: doctrine_mongodb, group: "data", label: Import dynamique }
 
@@ -53,11 +53,5 @@ services:
         tags:
             - { name: sonata.admin, manager_type: doctrine_mongodb, group: "data", label: Etiquettes }
 
-    admin.source: