Commit 62cb6983 authored by Sebastian Castro's avatar Sebastian Castro

Refactor Import and Import dynamic

parent a54fad02
......@@ -13,8 +13,8 @@ class ImportAdmin extends AbstractAdmin
protected function configureFormFields(FormMapper $formMapper)
{
$formMapper
->with("Importer des données depuis un fichier CSV",
["description" => "Les colonnes importantes du CSV sont les suivantes :
->with("Importer des données depuis un fichier CSV ou une API Json",
["description" => "Les colonnes/propriétés importantes sont les suivantes :
<ul>
<li><b>name</b> Le titre de la fiche</li>
<li><b>taxonomy</b> la liste des options séparées par des virgules. Exple: Alimentation, Restaurant
......@@ -27,19 +27,27 @@ class ImportAdmin extends AbstractAdmin
</ul>
Vous pouvez ensuite avoir n'importe quelles autres colonnes, elles seront importées. Veillez à faire concorder le nom des colonnes avec le nom des champs de votre formulaire"
])
// ->add('source', 'sonata_type_model', array(), array(
// 'class'=> 'Biopen\GeoDirectoryBundle\Document\Source',
// 'required' => true,
// 'label' => 'Source des données',
// 'mapped' => true))
->add('file', 'file', array('label' => 'Fichier à importer'))
->add('geocodeIfNecessary', null, array('required' => false, 'label' => 'Géocoder si élements sans latitude ni longitude'))
->add('sourceName', 'text', array(
'required' => true,
'label' => 'Nom de la source des données'))
->add('file', 'file', array('label' => 'Fichier à importer', 'required' => false))
->add('url', 'text', array('label' => 'Ou Url vers une API Json', 'required' => false))
->add('geocodeIfNecessary', null, array('required' => false, 'label' => 'Géocoder les élements sans latitude ni longitude à partir de leur adresse'))
->add('createMissingOptions', null, array('required' => false, 'label' => 'Créer les options manquantes'))
// ->add('parentCategoryToCreateOptions', 'sonata_type_model', array(
// 'class'=> 'Biopen\GeoDirectoryBundle\Document\Category',
// 'required' => false,
// 'btn_add' => false,
// 'label' => 'Catégorie parente pour créer les options manquantes',
// 'mapped' => true), array('admin_code' => 'admin.category'))
->add('createMissingOptions', null, array('required' => false, 'label' => 'Créer les options manquantes à partir des catégories renseignées dans chaque élément'))
->add('optionsToAddToEachElement', 'sonata_type_model', array(
'class'=> 'Biopen\GeoDirectoryBundle\Document\Option',
'required' => false,
'choices_as_values' => true,
'multiple' => true,
'btn_add' => false,
'label' => 'Options à ajouter à chaque élément importé'), array('admin_code' => 'admin.option'))
->end()
;
}
......
......@@ -8,12 +8,12 @@ use Sonata\AdminBundle\Datagrid\DatagridMapper;
use Sonata\AdminBundle\Form\FormMapper;
use Sonata\AdminBundle\Route\RouteCollection;
class SourceExternalAdmin extends AbstractAdmin
class ImportDynamicAdmin extends AbstractAdmin
{
public function getTemplate($name)
{
switch ($name) {
case 'edit': return '@BiopenAdmin/edit/edit_source_external.html.twig';
case 'edit': return '@BiopenAdmin/edit/edit_import_dynamic.html.twig';
break;
default : return parent::getTemplate($name);
break;
......@@ -23,8 +23,17 @@ class SourceExternalAdmin extends AbstractAdmin
protected function configureFormFields(FormMapper $formMapper)
{
$formMapper
->add('name', null, array('required' => false, 'label' => "Nom de la source"))
->add('url', null, array('required' => false, 'label' => "Url de l'api Json"))
->add('sourceName', 'text', array('required' => true, 'label' => 'Nom de la source '))
->add('url', 'text', array('label' => "Url de l'api Json", 'required' => true))
// ->add('geocodeIfNecessary', null, array('required' => false, 'label' => 'Géocoder les élements sans latitude ni longitude à partir de leur adresse'))
->add('createMissingOptions', null, array('required' => false, 'label' => 'Créer les options manquantes à partir des catégories renseignées dans chaque élément'))
->add('optionsToAddToEachElement', 'sonata_type_model', array(
'class'=> 'Biopen\GeoDirectoryBundle\Document\Option',
'required' => false,
'choices_as_values' => true,
'multiple' => true,
'btn_add' => false,
'label' => 'Options à ajouter à chaque élément importé'), array('admin_code' => 'admin.option'))
->add('refreshFrequencyInDays', null, array('required' => false, 'label' => "Fréquence de mise à jours des données en jours (laisser vide pour ne jamais mettre à jour automatiquement"));
}
......@@ -36,13 +45,13 @@ class SourceExternalAdmin extends AbstractAdmin
protected function configureDatagridFilters(DatagridMapper $datagridMapper)
{
$datagridMapper
->add('name')
->add('sourceName')
;
}
protected function configureListFields(ListMapper $listMapper)
{
$listMapper
->addIdentifier('name', null, array('label' => 'Nom de la source'))
->addIdentifier('sourceName', null, array('label' => 'Nom de la source'))
->add('lastRefresh', 'datetime', array('label' => 'Dernière synchronisation des données', 'format' => 'd/m/Y - H:i'))
->add('nextRefresh', 'date', array('label' => 'Prochaine synchronisation', 'template' => '@BiopenAdmin/partials/list_next_refresh.html.twig'))
->add('_action', 'actions', array(
......
<?php
namespace Biopen\GeoDirectoryBundle\Admin;
use Sonata\AdminBundle\Admin\AbstractAdmin;
use Sonata\AdminBundle\Datagrid\ListMapper;
use Sonata\AdminBundle\Datagrid\DatagridMapper;
use Sonata\AdminBundle\Form\FormMapper;
use Sonata\AdminBundle\Route\RouteCollection;
class SourceAdmin extends AbstractAdmin
{
protected function configureFormFields(FormMapper $formMapper)
{
$formMapper
->add('name', null, array('required' => false, 'label' => "Nom de la source"));
}
protected function configureDatagridFilters(DatagridMapper $datagridMapper)
{
$datagridMapper
->add('name')
;
}
protected function configureListFields(ListMapper $listMapper)
{
$listMapper
->addIdentifier('name', null, array('label' => 'Nom de la source'))
->add('_action', 'actions', array(
'actions' => array(
'edit' => array(),
'delete' => array(),
)
))
;
}
}
\ No newline at end of file
......@@ -20,19 +20,19 @@ class CheckExternalSourceToUpdateCommand extends GoGoAbstractCommand
}
protected function gogoExecute($em, InputInterface $input, OutputInterface $output)
{
$qb = $em->createQueryBuilder('BiopenGeoDirectoryBundle:SourceExternal');
{
$qb = $em->createQueryBuilder('BiopenGeoDirectoryBundle:ImportDynamic');
$sourcesToUpdate = $qb->field('refreshFrequencyInDays')->gt(0)
$dynamicImports = $qb->field('refreshFrequencyInDays')->gt(0)
->field('nextRefresh')->lte(new \DateTime())
->getQuery()->execute();
$importService = $this->getContainer()->get('biopen.element_import');
$this->log('CheckExternalSourceToUpdate : Nombre de sources à mettre à jour : ' . $sourcesToUpdate->count());
$this->log('CheckExternalSourceToUpdate : Nombre de sources à mettre à jour : ' . $dynamicImports->count());
foreach ($sourcesToUpdate as $key => $source)
foreach ($dynamicImports as $key => $import)
{
$this->log('Updating source : ' . $source->getName());
$this->log('Updating source : ' . $import->getSource()->getName());
$dataToImport = $importService->importJson($source);
}
}
......
......@@ -6,11 +6,11 @@ use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Biopen\SaasBundle\Command\GoGoAbstractCommand;
class GenerateElementsCommand extends ContainerAwareCommand
class GenerateElementsCommand extends GoGoAbstractCommand
{
protected function configure()
protected function gogoConfigure()
{
$this
->setName('app:elements:generate')
......@@ -21,10 +21,12 @@ class GenerateElementsCommand extends ContainerAwareCommand
;
}
protected function execute(InputInterface $input, OutputInterface $output)
protected function gogoExecute($em, InputInterface $input, OutputInterface $output)
{
$this->output = $output;
$this->getContainer()->get('biopen.random_creation_service')->generate($input->getArgument('number'), $input->getArgument('generateVotes'));
$output->writeln('Element générés !');
$this->log('Element générés !');
}
}
\ No newline at end of file
......@@ -17,27 +17,29 @@ class ImportSourceCommand extends GoGoAbstractCommand
$this
->setName('app:elements:importSource')
->setDescription('Check for updating external sources')
->addArgument('sourceName', InputArgument::REQUIRED, 'The name of the source');
->addArgument('sourceNameOrImportId', InputArgument::REQUIRED, 'The name of the source');
}
protected function gogoExecute($em, InputInterface $input, OutputInterface $output)
{
try {
$source = $em->getRepository('BiopenGeoDirectoryBundle:SourceExternal')->findOneByName($input->getArgument('sourceName'));
$this->output = $output;
$sourceNameOrId = $input->getArgument('sourceNameOrImportId');
$import = $em->getRepository('BiopenGeoDirectoryBundle:ImportDynamic')->find($sourceNameOrId);
if (!$import) $import = $em->getRepository('BiopenGeoDirectoryBundle:ImportDynamic')->findOneBySourceName($sourceNameOrId);
if (!$source)
if (!$import)
{
$message = "ERREUR pendant l'import : Aucune source avec pour nom " . $input->getArgument('sourceName') . " n'existe dans la base de donnée " . $input->getArgument('dbname');
$message = "ERREUR pendant l'import : Aucune source avec pour nom ou id " . $input->getArgument('sourceNameOrImportId') . " n'existe dans la base de donnée " . $input->getArgument('dbname');
$this->error($message);
return;
}
$this->log('Updating source ' . $source->getName() . ' for project ' . $input->getArgument('dbname') . ' begins...');
$this->log('Updating source ' . $import->getSourceName() . ' for project ' . $input->getArgument('dbname') . ' begins...');
$this->log('Downloading the data...');
$importService = $this->getContainer()->get('biopen.element_import');
$dataToImport = $importService->importJson($source, true);
$dataToImport = $importService->importJson($import, true);
$this->log('Data downloaded. ' . count($dataToImport) . ' elements to import...');
$count = $importService->import($dataToImport, $source);
$count = $importService->importData($dataToImport, $import);
$this->log('Updating source completed : ' . $count . ' elements successfully imported');
} catch (\Exception $e) {
$this->error($e->getMessage());
......
......@@ -23,13 +23,14 @@ class ImportAdminController extends Controller
private function executeImport($import)
{
$result = $this->get('biopen.element_import')->importCsv($import);
$result = $this->get('biopen.element_import')->startImport($import);
if ($result === null)
$this->addFlash('sonata_flash_error', "Un erreur s'est produite lors du chargement du fichier Csv. Vérifiez que le fichier est bien valide");
else
$this->addFlash('sonata_flash_success', 'Les ' . $result .' éléments ont été importés avec succès ');
}
// This method is just an overwrite of the SonataAdminCRUDController for calling the executeImport once the document is created
public function createAction()
{
......@@ -76,8 +77,7 @@ class ImportAdminController extends Controller
$this->admin->checkAccess('create', $object);
try {
$object = $this->admin->create($object);
$object = $this->admin->create($object);
$this->executeImport($object);
// redirect to create mode
......
......@@ -6,13 +6,13 @@ use Sonata\AdminBundle\Controller\CRUDController as Controller;
use Symfony\Component\HttpFoundation\RedirectResponse;
use Symfony\Component\HttpFoundation\Request;
class SourceExternalAdminController extends Controller
class ImportDynamicAdminController extends Controller
{
public function refreshAction()
{
$object = $this->admin->getSubject();
$this->get('biopen.async')->callCommand('app:elements:importSource', [$object->getName()]);
$this->get('biopen.async')->callCommand('app:elements:importSource', [$object->getId()]);
$this->addFlash('sonata_flash_success', "Les éléments sont en cours d'importation. Cela peut prendre plusieurs minutes.");
// $dataToImport = $this->get('biopen.element_import')->importJson($object);
......
......@@ -215,7 +215,7 @@ class Element
/**
* The source from where the element has been imported or created
*
* @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Source")
* @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Import")
*/
private $source;
......@@ -692,6 +692,23 @@ class Element
$this->privateData = null;
}
public function setCustomData($data, $privateProps)
{
$privateData = [];
foreach ($privateProps as $key => $prop) {
if (array_key_exists($prop, $data)) {
$privateData[$prop] = $data[$prop];
unset($data[$prop]);
}
}
if ($this->getData()) $data = array_merge($this->getData(), $data); // keeping also old data
$this->setData($data);
if ($this->getPrivateData()) $privateData = array_merge($this->getPrivateData(), $privateData); // keeping also old data
$this->setPrivateData($privateData);
}
/**
* Set status
*
......@@ -1398,10 +1415,10 @@ class Element
/**
* Set source
*
* @param Biopen\GeoDirectoryBundle\Document\Source $source
* @param Biopen\GeoDirectoryBundle\Document\Import $source
* @return $this
*/
public function setSource(\Biopen\GeoDirectoryBundle\Document\Source $source)
public function setSource(\Biopen\GeoDirectoryBundle\Document\Import $source)
{
$this->source = $source;
return $this;
......@@ -1410,7 +1427,7 @@ class Element
/**
* Get source
*
* @return Biopen\GeoDirectoryBundle\Document\Source $source
* @return Biopen\GeoDirectoryBundle\Document\Import $source
*/
public function getSource()
{
......
......@@ -10,6 +10,8 @@ use Biopen\CoreBundle\Document\AbstractFile;
/**
* @MongoDB\Document
* @Vich\Uploadable
* Import data into GoGoCarto. the data can imported through a static file, or via API url
* The Import can be made once for all (static import) or dynamically every X days (ImportDynamic)
*/
class Import extends AbstractFile
{
......@@ -22,20 +24,27 @@ class Import extends AbstractFile
private $id;
/**
* @var string
* @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Source", cascade={"persist"})
* @var string
* @MongoDB\Field(type="string")
*/
private $source;
public $sourceName;
/**
* @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Category", cascade={"persist"})
* @var string
* Url of API to get the data
* @MongoDB\Field(type="string")
*/
private $url;
/**
* @MongoDB\ReferenceOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Category")
*/
private $parentCategoryToCreateOptions = null;
/**
* @MongoDB\EmbedOne(targetDocument="Biopen\GeoDirectoryBundle\Document\Option")
* @MongoDB\ReferenceMany(targetDocument="Biopen\GeoDirectoryBundle\Document\Option")
*/
private $optionTemplate;
private $optionsToAddToEachElement = [];
/**
* @MongoDB\Field(type="bool")
......@@ -49,6 +58,8 @@ class Import extends AbstractFile
public function __construct() {}
public function isDynamicImport() { return false; }
/**
* Get id
*
......@@ -82,26 +93,26 @@ class Import extends AbstractFile
}
/**
* Set optionTemplate
* Set url
*
* @param Biopen\GeoDirectoryBundle\Document\Option $optionTemplate
* @param string $url
* @return $this
*/
public function setOptionTemplate(\Biopen\GeoDirectoryBundle\Document\Option $optionTemplate)
public function setUrl($url)
{
$this->optionTemplate = $optionTemplate;
$this->url = $url;
return $this;
}
/**
* Get optionTemplate
* Get url
*
* @return Biopen\GeoDirectoryBundle\Document\Option $optionTemplate
* @return string $url
*/
public function getOptionTemplate()
public function getUrl()
{
return $this->optionTemplate;
}
return $this->url;
}
/**
* Set createMissingOptions
......@@ -148,24 +159,54 @@ class Import extends AbstractFile
}
/**
* Set source
* Add optionsToAddToEachElement
*
* @param Biopen\GeoDirectoryBundle\Document\Option $optionsToAddToEachElement
*/
public function addOptionsToAddToEachElement(\Biopen\GeoDirectoryBundle\Document\Option $optionsToAddToEachElement)
{
$this->optionsToAddToEachElement[] = $optionsToAddToEachElement;
}
/**
* Remove optionsToAddToEachElement
*
* @param Biopen\GeoDirectoryBundle\Document\Option $optionsToAddToEachElement
*/
public function removeOptionsToAddToEachElement(\Biopen\GeoDirectoryBundle\Document\Option $optionsToAddToEachElement)
{
$this->optionsToAddToEachElement->removeElement($optionsToAddToEachElement);
}
/**
* Get optionsToAddToEachElement
*
* @return \Doctrine\Common\Collections\Collection $optionsToAddToEachElement
*/
public function getOptionsToAddToEachElement()
{
return $this->optionsToAddToEachElement;
}
/**
* Set sourceName
*
* @param Biopen\GeoDirectoryBundle\Document\Source $source
* @param string $sourceName
* @return $this
*/
public function setSource(\Biopen\GeoDirectoryBundle\Document\Source $source)
public function setSourceName($sourceName)
{
$this->source = $source;
$this->sourceName = $sourceName;
return $this;
}
/**
* Get source
* Get sourceName
*
* @return Biopen\GeoDirectoryBundle\Document\Source $source
* @return string $sourceName
*/
public function getSource()
public function getSourceName()
{
return $this->source;
return $this->sourceName;
}
}
......@@ -10,14 +10,8 @@ use \Datetime;
*
* @MongoDB\Document
*/
class SourceExternal extends Source
class ImportDynamic extends Import
{
/**
* @var string
* @MongoDB\Field(type="string")
*/
private $url;
/**
* @var string
* @MongoDB\Field(type="int")
......@@ -38,7 +32,7 @@ class SourceExternal extends Source
*/
private $nextRefresh = null;
public function isExternalsource() { return true; }
public function isDynamicImport() { return true; }
public function updateNextRefreshDate()
{
......@@ -52,28 +46,6 @@ class SourceExternal extends Source
}
}
/**
* Set url
*
* @param string $url
* @return $this
*/
public function setUrl($url)
{
$this->url = $url;
return $this;
}
/**
* Get url
*
* @return string $url
*/
public function getUrl()
{
return $this->url;
}
/**
* Set refreshFrequencyInDays
*
......
......@@ -188,7 +188,7 @@ class Option
public function __toString()
{
return "(Option) " . $this->getName();
return $this->getName();
}
public function getNameWithParent()
......
<?php
namespace Biopen\GeoDirectoryBundle\Document;
use Doctrine\ODM\MongoDB\Mapping\Annotations as MongoDB;
/**
* Source
*
* @MongoDB\Document
*/
class Source
{
/**
* @var int
* @MongoDB\Id(strategy="INCREMENT")
*/
protected $id;
/**
* @var string
* @MongoDB\Field(type="string")
*/
protected $name;
public function __toString() { return $this->name; }
public function isExternalsource() { return false; }
/**
* Get id
*
* @return int_id $id
*/
public function getId()
{
return $this->id;
}
/**
* Set name
*
* @param string $name
* @return $this
*/
public function setName($name)
{
$this->name = $name;
return $this;
}
/**
* Get name
*
* @return string $name
*/
public function getName()
{
return $this->name;
}
}
\ No newline at end of file
......@@ -42,8 +42,8 @@ services:
- { name: sonata.admin, manager_type: doctrine_mongodb, group: "data", label: Import }
admin.source_external:
class: Biopen\GeoDirectoryBundle\Admin\SourceExternalAdmin
arguments: [~, Biopen\GeoDirectoryBundle\Document\SourceExternal, 'BiopenGeoDirectoryBundle:Admin/SourceExternalAdmin' ]
class: Biopen\GeoDirectoryBundle\Admin\ImportDynamicAdmin
arguments: [~, Biopen\GeoDirectoryBundle\Document\ImportDynamic, 'BiopenGeoDirectoryBundle:Admin/ImportDynamicAdmin' ]
tags:
- { name: sonata.admin, manager_type: doctrine_mongodb, group: "data", label: Import dynamique }
......@@ -53,11 +53,5 @@ services:
tags:
- { name: sonata.admin, manager_type: doctrine_mongodb, group: "data", label: Etiquettes }
admin.source:
class: Biopen\GeoDirectoryBundle\Admin\SourceAdmin
arguments: [~, Biopen\GeoDirectoryBundle\Document\Source, '' ]
tags:
- { name: sonata.admin, manager_type: doctrine_mongodb, group: "Hide", label: Source }
......@@ -5,9 +5,11 @@
{% if field_description.options.choices is defined %}
<div style="font-style: italic;font-size: .9em;">
{% for key, optionValue in value %}
{% if field_description.options.choices[optionValue.optionId] is defined %}
{% set option = field_description.options.choices[optionValue.optionId] %}
{{ option.nameShort is defined ? option.nameShort : option.name }}
{% if not loop.last %}, {% endif %}
{% endif %}
{% endfor %}
</div>
{% endif %}
......
......@@ -95,20 +95,7 @@ class ElementFormService
$config = $em->getRepository('BiopenCoreBundle:Configuration')->findConfiguration();
$privateProp = $config->getApi()->getPublicApiPrivateProperties();
$data = $request->get('data');
$privateData = [];
foreach ($privateProp as $key => $prop) {
if (array_key_exists($prop, $data)) {
$privateData[$prop] = $data[$prop];
unset($data[$prop]);
}
}
if ($element->getData()) $data = array_merge($element->getData(), $data); // keeping also old data
$element->setData($data);
if ($element->getPrivateData()) $privateData = array_merge($element->getPrivateData(), $privateData); // keeping also old data
$element->setPrivateData($privateData);
$element->setCustomData($request->get('data'), $privateProp);
}
private function updateWebsiteUrl($element)
......
......@@ -27,10 +27,12 @@ class ElementImportService
private $elementActionService;
protected $createMissingOptions;
protected $optionsToAddToEachElement;
protected $parentCategoryToCreateMissingOptions;
protected $missingOptionDefaultAttributesForCreate;
protected $coreFields = ['id', 'name', 'taxonomy', 'streetAddress', 'addressLocality', 'postalCode', 'addressCountry', 'email', 'latitude', 'longitude', 'images', 'owner', 'source'];
protected $privateDataProps;
/**
* Constructor
*/
......@@ -43,6 +45,11 @@ class ElementImportService
$this->currentRow = [];
}
public function startImport($import) {
if ($import->getUrl()) return $this->importJson($import);
else return $this->importCsv($import);
}
public function importCsv($import)
{
$fileName = $import->calculateFilePath();
......@@ -51,18 +58,13 @@ class ElementImportService
$data = $this->converter->convert($fileName, ',');
if ($data === null) return null;
return $this->import($data,
$import->getSource(),
$import->getGeocodeIfNecessary(),
$import->getCreateMissingOptions(),
$import->getParentCategoryToCreateOptions());
return $this->importData($data, $import);
}