Improve element import flow

parent 1aef2621
...@@ -17,7 +17,8 @@ class AppExtension extends AbstractExtension ...@@ -17,7 +17,8 @@ class AppExtension extends AbstractExtension
public function getFilters() public function getFilters()
{ {
return array( return array(
new TwigFilter('json_decode', array($this, 'jsonDecode')), new TwigFilter('json_decode', array($this, 'jsonDecode')),
new TwigFilter('values', array($this, 'values')),
); );
} }
...@@ -26,6 +27,11 @@ class AppExtension extends AbstractExtension ...@@ -26,6 +27,11 @@ class AppExtension extends AbstractExtension
return json_decode($value); return json_decode($value);
} }
public function values($value)
{
return array_values($value);
}
public function getFunctions() public function getFunctions()
{ {
return array( return array(
...@@ -43,11 +49,11 @@ class AppExtension extends AbstractExtension ...@@ -43,11 +49,11 @@ class AppExtension extends AbstractExtension
public function getNewMessagesCount() public function getNewMessagesCount()
{ {
return count($this->dm->getRepository('BiopenCoreBundle:GoGoLog')->findBy(['type' => 'update', 'hidden' => false])); return count($this->dm->getRepository('BiopenCoreBundle:GoGoLog')->findBy(['type' => 'update', 'hidden' => false]));
} }
public function getErrorsCount() public function getErrorsCount()
{ {
return count($this->dm->getRepository('BiopenCoreBundle:GoGoLog')->findBy(['level' => 'error', 'hidden' => false])); return count($this->dm->getRepository('BiopenCoreBundle:GoGoLog')->findBy(['level' => 'error', 'hidden' => false]));
} }
} }
\ No newline at end of file
...@@ -50,21 +50,35 @@ class ImportDynamicAdmin extends ImportAbstractAdmin ...@@ -50,21 +50,35 @@ class ImportDynamicAdmin extends ImportAbstractAdmin
'label' => 'Catégories à ajouter à chaque élément importé'), array('admin_code' => 'admin.option')) 'label' => 'Catégories à ajouter à chaque élément importé'), array('admin_code' => 'admin.option'))
->add('needToHaveOptionsOtherThanTheOnesAddedToEachElements', null, array('required' => false, 'label' => 'Les éléments importés doivent contenir au moins une catégorie en dehors de celles ajoutées manuellement ci-dessus', 'label_attr' => ['title' => "Sans prendre en compte les catégories ajoutés via le champs \"Catégories à ajouter à chaque élément importé\", si les éléments importés n'ont pas de catégories, ils seront marqués comme \"Modération aucune catégorie renseignée\""])) ->add('needToHaveOptionsOtherThanTheOnesAddedToEachElements', null, array('required' => false, 'label' => 'Les éléments importés doivent contenir au moins une catégorie en dehors de celles ajoutées manuellement ci-dessus', 'label_attr' => ['title' => "Sans prendre en compte les catégories ajoutés via le champs \"Catégories à ajouter à chaque élément importé\", si les éléments importés n'ont pas de catégories, ils seront marqués comme \"Modération aucune catégorie renseignée\""]))
->add('fieldToCheckElementHaveBeenUpdated', null, array('required' => false, 'label' => "Nom de l'attribut à comparer pour la mise à jour", 'label_attr' => ['title' => "Lorsqu'on met à jour une source, certains des éléments à importer existent déjà dans notre base de donnée. Vous pouvez renseigner ici un champs qui permettra de comparer si l'élément à été mis à jour au sein de la source depuis le dernier import. Exple de champ: updatedAt, date_maj etc... (laisser vide pour mettre à jour les éléments à chaque fois)"])) ->add('fieldToCheckElementHaveBeenUpdated', null, array('required' => false, 'label' => "Nom de l'attribut à comparer pour la mise à jour", 'label_attr' => ['title' => "Lorsqu'on met à jour une source, certains des éléments à importer existent déjà dans notre base de donnée. Vous pouvez renseigner ici un champs qui permettra de comparer si l'élément à été mis à jour au sein de la source depuis le dernier import. Exple de champ: updatedAt, date_maj etc... (laisser vide pour mettre à jour les éléments à chaque fois)"]))
->end() ->end();
->with('Historique', array('class' => 'col-sm-12')) if ($this->getSubject()->getId())
->add('logs', 'hidden', array('attr' => ['class' => 'gogo-display-logs'], 'mapped' => false)) {
->end() $formMapper->with('Historique', array('class' => 'col-sm-12'))
->end() ->add('logs', 'hidden', array('attr' => ['class' => 'gogo-display-logs'], 'mapped' => false))
->tab('Table de correspondance des champs') ->end();
->with('Transformer les données à importer') }
->add('ontologyMapping', 'hidden', array('attr' => ['class' => 'gogo-mapping-ontology', 'data-form-props' => $formProperties, 'data-props' => $elementProperties])) $formMapper->end();
->end()
->end() if ($this->getSubject()->getId())
->tab('Table de correspondance des catégories') {
->with('Faites correspondre les catégories') $formMapper
->add('taxonomyMapping', 'hidden', array('attr' => ['class' => 'gogo-mapping-taxonomy', 'data-options' => $optionsList])) ->tab('Table de correspondance des champs')
->end() ->with('Transformer les données à importer')
->end() ->add('ontologyMapping', 'hidden', array('attr' => ['class' => 'gogo-mapping-ontology', 'data-form-props' => $formProperties, 'data-props' => $elementProperties]))
->end()
->end();
if (count($this->getSubject()->getOntologyMapping()) > 0)
{
$formMapper->tab('Table de correspondance des catégories')
->with('Faites correspondre les catégories')
->add('taxonomyMapping', 'hidden', array('attr' => ['class' => 'gogo-mapping-taxonomy', 'data-options' => $optionsList]))
->end()
->end();
}
}
$formMapper
->tab('Aide') ->tab('Aide')
->with("Aide", ['box_class' => 'box box-default', "description" => $this->getInstructions('13154fa0-13c2-41f1-a4ad-e04c35c86e89')]) ->with("Aide", ['box_class' => 'box box-default', "description" => $this->getInstructions('13154fa0-13c2-41f1-a4ad-e04c35c86e89')])
->end() ->end()
......
...@@ -128,4 +128,77 @@ class ImportDynamicAdminController extends Controller ...@@ -128,4 +128,77 @@ class ImportDynamicAdminController extends Controller
'object' => $object, 'object' => $object,
), null); ), null);
} }
/**
* Overwrite Sonata CRud Controller
*/
public function createAction()
{
$request = $this->getRequest();
// the key used to lookup the template
$templateKey = 'edit';
$this->admin->checkAccess('create');
$class = new \ReflectionClass($this->admin->hasActiveSubClass() ? $this->admin->getActiveSubClass() : $this->admin->getClass());
$object = $this->admin->getNewInstance();
$this->admin->setSubject($object);
$form = $this->admin->getForm();
$form->setData($object);
$form->handleRequest($request);
if ($form->isSubmitted()) {
//TODO: remove this check for 4.0
if (method_exists($this->admin, 'preValidate')) {
$this->admin->preValidate($object);
}
$isFormValid = $form->isValid();
// persist if the form was valid and if in preview mode the preview was approved
if ($isFormValid && (!$this->isInPreviewMode($request) || $this->isPreviewApproved($request))) {
try {
$object = $this->admin->create($object);
$this->addFlash('sonata_flash_success', "Import créé avec succès. Vous pouvez maintenant cliquez sur 'Lire les Données' pour charger la liste des champs des éléments à importer");
$url = $this->admin->generateUrl('edit', ['id' => $object->getId()]) . "#tab_2";
return $this->redirect($url);
} catch (ModelManagerException $e) {
$this->handleModelManagerException($e);
$isFormValid = false;
}
}
// show an error message if the form failed validation
if (!$isFormValid) {
if (!$this->isXmlHttpRequest()) {
$this->addFlash(
'sonata_flash_error',
$this->trans(
'flash_create_error',
array('%name%' => $this->escapeHtml($this->admin->toString($object))),
'SonataAdminBundle'
)
);
}
} elseif ($this->isPreviewRequested()) {
// pick the preview template if the form was valid and preview was requested
$templateKey = 'preview';
$this->admin->getShow();
}
}
$view = $form->createView();
// set the theme for the current Admin Form
$this->get('twig')->getExtension('form')->renderer->setTheme($view, $this->admin->getFormTheme());
return $this->render($this->admin->getTemplate($templateKey), array(
'action' => 'create',
'form' => $view,
'object' => $object,
), null);
}
} }
\ No newline at end of file
...@@ -8,17 +8,17 @@ use Vich\UploaderBundle\Mapping\Annotation as Vich; ...@@ -8,17 +8,17 @@ use Vich\UploaderBundle\Mapping\Annotation as Vich;
use Biopen\CoreBundle\Document\AbstractFile; use Biopen\CoreBundle\Document\AbstractFile;
abstract class ImportState abstract class ImportState
{ {
const Started = "started"; const Started = "started";
const Downloading = "downloading"; const Downloading = "downloading";
const InProgress = "in_progress"; const InProgress = "in_progress";
const Completed = "completed"; const Completed = "completed";
const Errors = "errors"; const Errors = "errors";
const Failed = "failed"; const Failed = "failed";
} }
/** /**
* @MongoDB\Document * @MongoDB\Document
* @Vich\Uploadable * @Vich\Uploadable
* Import data into GoGoCarto. the data can imported through a static file, or via API url * Import data into GoGoCarto. the data can imported through a static file, or via API url
* The Import can be made once for all (static import) or dynamically every X days (ImportDynamic) * The Import can be made once for all (static import) or dynamically every X days (ImportDynamic)
...@@ -33,18 +33,18 @@ class Import extends AbstractFile ...@@ -33,18 +33,18 @@ class Import extends AbstractFile
/** /**
* @var int * @var int
* @MongoDB\Id(strategy="INCREMENT") * @MongoDB\Id(strategy="INCREMENT")
*/ */
private $id; private $id;
/** /**
* @var string * @var string
* @MongoDB\Field(type="string") * @MongoDB\Field(type="string")
*/ */
public $sourceName; public $sourceName;
/** /**
* @var string * @var string
* Url of API to get the data * Url of API to get the data
* @MongoDB\Field(type="string") * @MongoDB\Field(type="string")
*/ */
...@@ -71,7 +71,7 @@ class Import extends AbstractFile ...@@ -71,7 +71,7 @@ class Import extends AbstractFile
private $needToHaveOptionsOtherThanTheOnesAddedToEachElements = false; private $needToHaveOptionsOtherThanTheOnesAddedToEachElements = false;
/** /**
* @var string * @var string
* @MongoDB\Field(type="string") * @MongoDB\Field(type="string")
*/ */
public $fieldToCheckElementHaveBeenUpdated; public $fieldToCheckElementHaveBeenUpdated;
...@@ -79,7 +79,7 @@ class Import extends AbstractFile ...@@ -79,7 +79,7 @@ class Import extends AbstractFile
/** /**
* @MongoDB\Field(type="bool") * @MongoDB\Field(type="bool")
*/ */
private $geocodeIfNecessary = false; private $geocodeIfNecessary = false;
/** /**
* @MongoDB\ReferenceMany(targetDocument="Biopen\CoreBundle\Document\GoGoLog", cascade={"all"}) * @MongoDB\ReferenceMany(targetDocument="Biopen\CoreBundle\Document\GoGoLog", cascade={"all"})
...@@ -88,7 +88,7 @@ class Import extends AbstractFile ...@@ -88,7 +88,7 @@ class Import extends AbstractFile
/** /**
* State of the import when processing. Type of ImportState * State of the import when processing. Type of ImportState
* When processing import, this variable is being updated in realtime, so the client can follow * When processing import, this variable is being updated in realtime, so the client can follow
* the state of the import also in realtime * the state of the import also in realtime
* @MongoDB\Field(type="string") * @MongoDB\Field(type="string")
*/ */
...@@ -100,26 +100,26 @@ class Import extends AbstractFile ...@@ -100,26 +100,26 @@ class Import extends AbstractFile
*/ */
private $currMessage; private $currMessage;
/** /**
* After importing some Data, if the user hard delete some elements, their ids will be remembered * After importing some Data, if the user hard delete some elements, their ids will be remembered
* so next time we do not import them again * so next time we do not import them again
* *
* @MongoDB\Field(type="collection") * @MongoDB\Field(type="collection")
*/ */
private $idsToIgnore = []; private $idsToIgnore = [];
/** /**
* @MongoDB\Field(type="hash") * @MongoDB\Field(type="hash")
*/ */
private $ontologyMapping = []; private $ontologyMapping = [];
/** /**
* @MongoDB\Field(type="hash") * @MongoDB\Field(type="hash")
*/ */
private $taxonomyMapping = []; private $taxonomyMapping = [];
public function __construct() { public function __construct() {
$this->logs = new \Doctrine\Common\Collections\ArrayCollection();; $this->logs = new \Doctrine\Common\Collections\ArrayCollection();;
} }
...@@ -128,10 +128,15 @@ class Import extends AbstractFile ...@@ -128,10 +128,15 @@ class Import extends AbstractFile
public function isDynamicImport() { return false; } public function isDynamicImport() { return false; }
public function addIdToIgnore($id) public function addIdToIgnore($id)
{ {
$this->idsToIgnore[] = $id; $this->idsToIgnore[] = $id;
} }
public function isCategoriesFieldMapped()
{
return in_array('categories', array_values($this->getOntologyMapping()));
}
/** /**
* Get id * Get id
...@@ -185,7 +190,7 @@ class Import extends AbstractFile ...@@ -185,7 +190,7 @@ class Import extends AbstractFile
public function getUrl() public function getUrl()
{ {
return $this->url; return $this->url;
} }
/** /**
* Set createMissingOptions * Set createMissingOptions
...@@ -309,7 +314,7 @@ class Import extends AbstractFile ...@@ -309,7 +314,7 @@ class Import extends AbstractFile
* @return \Doctrine\Common\Collections\Collection $logs * @return \Doctrine\Common\Collections\Collection $logs
*/ */
public function getLogs() public function getLogs()
{ {
$logs = is_array($this->logs) ? $this->logs : $this->logs->toArray(); $logs = is_array($this->logs) ? $this->logs : $this->logs->toArray();
usort( $logs, function ($a, $b) { return $b->getCreatedAt()->getTimestamp() - $a->getCreatedAt()->getTimestamp(); }); usort( $logs, function ($a, $b) { return $b->getCreatedAt()->getTimestamp() - $a->getCreatedAt()->getTimestamp(); });
return $logs; return $logs;
...@@ -359,27 +364,27 @@ class Import extends AbstractFile ...@@ -359,27 +364,27 @@ class Import extends AbstractFile
return $this->currMessage; return $this->currMessage;
} }
/** /**
* Set idsToIgnore * Set idsToIgnore
* *
* @param collection $idsToIgnore * @param collection $idsToIgnore
* @return $this * @return $this
*/ */
public function setIdsToIgnore($idsToIgnore) public function setIdsToIgnore($idsToIgnore)
{ {
$this->idsToIgnore = $idsToIgnore; $this->idsToIgnore = $idsToIgnore;
return $this; return $this;
} }
/** /**
* Get idsToIgnore * Get idsToIgnore
* *
* @return collection $idsToIgnore * @return collection $idsToIgnore
*/ */
public function getIdsToIgnore() public function getIdsToIgnore()
{ {
return $this->idsToIgnore; return $this->idsToIgnore;
} }
/** /**
* Set needToHaveOptionsOtherThanTheOnesAddedToEachElements * Set needToHaveOptionsOtherThanTheOnesAddedToEachElements
......
...@@ -2,25 +2,30 @@ ...@@ -2,25 +2,30 @@
{% set object = form.vars.sonata_admin.admin.subject %} {% set object = form.vars.sonata_admin.admin.subject %}
{% set mapping = object.ontologyMapping %} {% set mapping = object.ontologyMapping %}
<table class="table">
<thead> {% if mapping|length == 0 %}
<tr> <div class="alert alert-info">Veuillez cliquer sur le boutton "Lire les données" afin de charger les attributs des données à importer</div>
<th>Attribut d'origine</th> {% else %}
<th style="width:20px"></th> <table class="table">
<th>à transformer en</th> <thead>
</tr>
</thead>
<tbody class="table-striped">
{% for originName, mappedName in mapping %}
<tr> <tr>
<td class="original">{{originName}}</td> <th>Attribut d'origine</th>
<td><i class="arrow-icon fa fa-arrow-circle-right"></i></td> <th style="width:20px"></th>
<td class="mapped"><input type="text" name="ontology[{{originName}}]" class="form-control property-selector" value={{mappedName}} /></td> <th>à transformer en</th>
</tr> </tr>
{% endfor %} </thead>
</tbody> <tbody class="table-striped">
{% for originName, mappedName in mapping %}
<tr>
<td class="original">{{originName}}</td>
<td><i class="arrow-icon fa fa-arrow-circle-right"></i></td>
<td class="mapped"><input type="text" name="ontology[{{originName}}]" class="form-control property-selector" value={{mappedName}} /></td>
</tr>
{% endfor %}
</tbody>
</table> </table>
{% endif %}
<style> <style>
.arrow-icon { font-size: 18px; } .arrow-icon { font-size: 18px; }
......
...@@ -2,25 +2,30 @@ ...@@ -2,25 +2,30 @@
{% set object = form.vars.sonata_admin.admin.subject %} {% set object = form.vars.sonata_admin.admin.subject %}
{% set mapping = object.taxonomyMapping %} {% set mapping = object.taxonomyMapping %}
<table class="table"> {% if object.isCategoriesFieldMapped %}
<thead> <table class="table">
<tr> <thead>
<th>Catégorie d'origine</th>
<th style="width:20px"></th>
<th>à transformer en</th>
</tr>
</thead>
<tbody class="table-striped">
{% for originName, mappedName in mapping %}
<tr> <tr>
<td class="original">{{originName}}</td> <th>Catégories lues depuis le fichier d'origine</th>
<td><i class="fa fa-arrow-circle-right"></i></td> <th style="width:20px"></th>
<td class="mapped"><input type="text" name="taxonomy[{{originName}}]" class="form-control category-selector" value={{mappedName}} /></td> <th>Catégories du site</th>
</tr> </tr>
{% endfor %} </thead>
</tbody> <tbody class="table-striped">
{% for originName, mappedName in mapping %}
<tr>
<td class="original">{{originName}}</td>
<td><i class="fa fa-arrow-circle-right"></i></td>
<td class="mapped"><input type="text" name="taxonomy[{{originName}}]" class="form-control category-selector" value={{mappedName}} /></td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<div class="alert alert-info">Veuillez d'abords sélectionner le champs correspondant aux catégories dans l'onglet 'Table de correspondance des champs' puis cliquer sur 'Sauvegarder'</div>
{% endif %}
</table>
<script> <script>
jQuery(document).ready(function() { jQuery(document).ready(function() {
......
...@@ -7,9 +7,18 @@ ...@@ -7,9 +7,18 @@
{% if object.lastRefresh %} {% if object.lastRefresh %}
<p>Dernière mise à jour : {{ object.lastRefresh|date('d/m/Y') }}</p> <p>Dernière mise à jour : {{ object.lastRefresh|date('d/m/Y') }}</p>
{% endif %} {% endif %}
{% set class = "btn-primary" %}
<button type="submit" class="btn btn-success" name="submit"><i class="fa fa-save" aria-hidden="true"></i>Sauvegarder</button> <button type="submit" class="btn btn-success" name="submit"><i class="fa fa-save" aria-hidden="true"></i>Sauvegarder</button>
{% include '@BiopenAdmin/partials/list__action_refresh.html.twig' %}
{% include '@BiopenAdmin/partials/list__action_refresh.html.twig' with {'class': 'btn-primary'} %}
{% set class = object.ontologyMapping|length > 0 ? 'btn-default' : 'btn-primary' %}
{% if object.id != "" %}
<a href="{{ admin.generateObjectUrl('collect', object) }}" class="btn {{ class }} view_link">
<i class="fa fa-eye" aria-hidden="true"></i>
Lire les données
</a>
{% endif %}
<a href="{{ admin.generateObjectUrl('list', object) }}" class="btn btn-default view_link"> <a href="{{ admin.generateObjectUrl('list', object) }}" class="btn btn-default view_link">
<i class="fa fa-list" aria-hidden="true"></i>Retour à la liste <i class="fa fa-list" aria-hidden="true"></i>Retour à la liste
</a> </a>
......
...@@ -10,12 +10,8 @@ file that was distributed with this source code. ...@@ -10,12 +10,8 @@ file that was distributed with this source code.
#} #}
{% if admin.isGranted('VIEW', object) and object.id != "" %} {% if admin.isGranted('VIEW', object) and object.id != "" %}
{% if object.ontologyMapping|length == 0 %}
<a href="{{ admin.generateObjectUrl('collect', object) }}" class="btn {{ class|default('btn-sm btn-primary') }} view_link"> {% if object.ontologyMapping|length > 0 %}
<i class="fa fa-eye" aria-hidden="true"></i>
Lire les données
</a>
{% else %}
<a href="{{ admin.generateObjectUrl('refresh', object) }}" class="btn {{ class|default('btn-sm btn-primary') }} view_link"> <a href="{{ admin.generateObjectUrl('refresh', object) }}" class="btn {{ class|default('btn-sm btn-primary') }} view_link">
{% if object.lastRefresh %} {% if object.lastRefresh %}
<i class="fa fa-refresh" aria-hidden="true"></i> <i class="fa fa-refresh" aria-hidden="true"></i>
......
...@@ -26,6 +26,16 @@ class ElementImportMappingService ...@@ -26,6 +26,16 @@ class ElementImportMappingService
protected $parentCategoryIdToCreateMissingOptions; protected $parentCategoryIdToCreateMissingOptions;
protected $em; protected $em;
protected $coreFields = ['id', 'name', 'categories', 'streetAddress', 'addressLocality', 'postalCode', 'addressCountry', 'latitude', 'longitude', 'images', 'owner', 'source']; protected $coreFields = ['id', 'name', 'categories', 'streetAddress', 'addressLocality', 'postalCode', 'addressCountry', 'latitude', 'longitude', 'images', 'owner', 'source'];
protected $mappedCoreFields = [
'title' => 'name',
'taxonomy' => 'categories',
'address' => 'streetAddress',
'city' => 'addressLocatily',
'postcode' => 'postalCode',
'country' => 'addressCountry',