Commit 46c5056b authored by Sebastian Castro's avatar Sebastian Castro

Improve element import flow

parent 1aef2621
......@@ -17,7 +17,8 @@ class AppExtension extends AbstractExtension
public function getFilters()
{
return array(
new TwigFilter('json_decode', array($this, 'jsonDecode')),
new TwigFilter('json_decode', array($this, 'jsonDecode')),
new TwigFilter('values', array($this, 'values')),
);
}
......@@ -26,6 +27,11 @@ class AppExtension extends AbstractExtension
return json_decode($value);
}
public function values($value)
{
return array_values($value);
}
public function getFunctions()
{
return array(
......@@ -43,11 +49,11 @@ class AppExtension extends AbstractExtension
public function getNewMessagesCount()
{
return count($this->dm->getRepository('BiopenCoreBundle:GoGoLog')->findBy(['type' => 'update', 'hidden' => false]));
return count($this->dm->getRepository('BiopenCoreBundle:GoGoLog')->findBy(['type' => 'update', 'hidden' => false]));
}
public function getErrorsCount()
{
return count($this->dm->getRepository('BiopenCoreBundle:GoGoLog')->findBy(['level' => 'error', 'hidden' => false]));
return count($this->dm->getRepository('BiopenCoreBundle:GoGoLog')->findBy(['level' => 'error', 'hidden' => false]));
}
}
\ No newline at end of file
......@@ -50,21 +50,35 @@ class ImportDynamicAdmin extends ImportAbstractAdmin
'label' => 'Catégories à ajouter à chaque élément importé'), array('admin_code' => 'admin.option'))
->add('needToHaveOptionsOtherThanTheOnesAddedToEachElements', null, array('required' => false, 'label' => 'Les éléments importés doivent contenir au moins une catégorie en dehors de celles ajoutées manuellement ci-dessus', 'label_attr' => ['title' => "Sans prendre en compte les catégories ajoutés via le champs \"Catégories à ajouter à chaque élément importé\", si les éléments importés n'ont pas de catégories, ils seront marqués comme \"Modération aucune catégorie renseignée\""]))
->add('fieldToCheckElementHaveBeenUpdated', null, array('required' => false, 'label' => "Nom de l'attribut à comparer pour la mise à jour", 'label_attr' => ['title' => "Lorsqu'on met à jour une source, certains des éléments à importer existent déjà dans notre base de donnée. Vous pouvez renseigner ici un champs qui permettra de comparer si l'élément à été mis à jour au sein de la source depuis le dernier import. Exple de champ: updatedAt, date_maj etc... (laisser vide pour mettre à jour les éléments à chaque fois)"]))
->end()
->with('Historique', array('class' => 'col-sm-12'))
->add('logs', 'hidden', array('attr' => ['class' => 'gogo-display-logs'], 'mapped' => false))
->end()
->end()
->tab('Table de correspondance des champs')
->with('Transformer les données à importer')
->add('ontologyMapping', 'hidden', array('attr' => ['class' => 'gogo-mapping-ontology', 'data-form-props' => $formProperties, 'data-props' => $elementProperties]))
->end()
->end()
->tab('Table de correspondance des catégories')
->with('Faites correspondre les catégories')
->add('taxonomyMapping', 'hidden', array('attr' => ['class' => 'gogo-mapping-taxonomy', 'data-options' => $optionsList]))
->end()
->end()
->end();
if ($this->getSubject()->getId())
{
$formMapper->with('Historique', array('class' => 'col-sm-12'))
->add('logs', 'hidden', array('attr' => ['class' => 'gogo-display-logs'], 'mapped' => false))
->end();
}
$formMapper->end();
if ($this->getSubject()->getId())
{
$formMapper
->tab('Table de correspondance des champs')
->with('Transformer les données à importer')
->add('ontologyMapping', 'hidden', array('attr' => ['class' => 'gogo-mapping-ontology', 'data-form-props' => $formProperties, 'data-props' => $elementProperties]))
->end()
->end();
if (count($this->getSubject()->getOntologyMapping()) > 0)
{
$formMapper->tab('Table de correspondance des catégories')
->with('Faites correspondre les catégories')
->add('taxonomyMapping', 'hidden', array('attr' => ['class' => 'gogo-mapping-taxonomy', 'data-options' => $optionsList]))
->end()
->end();
}
}
$formMapper
->tab('Aide')
->with("Aide", ['box_class' => 'box box-default', "description" => $this->getInstructions('13154fa0-13c2-41f1-a4ad-e04c35c86e89')])
->end()
......
......@@ -128,4 +128,77 @@ class ImportDynamicAdminController extends Controller
'object' => $object,
), null);
}
/**
* Overwrite Sonata CRud Controller
*/
public function createAction()
{
$request = $this->getRequest();
// the key used to lookup the template
$templateKey = 'edit';
$this->admin->checkAccess('create');
$class = new \ReflectionClass($this->admin->hasActiveSubClass() ? $this->admin->getActiveSubClass() : $this->admin->getClass());
$object = $this->admin->getNewInstance();
$this->admin->setSubject($object);
$form = $this->admin->getForm();
$form->setData($object);
$form->handleRequest($request);
if ($form->isSubmitted()) {
//TODO: remove this check for 4.0
if (method_exists($this->admin, 'preValidate')) {
$this->admin->preValidate($object);
}
$isFormValid = $form->isValid();
// persist if the form was valid and if in preview mode the preview was approved
if ($isFormValid && (!$this->isInPreviewMode($request) || $this->isPreviewApproved($request))) {
try {
$object = $this->admin->create($object);
$this->addFlash('sonata_flash_success', "Import créé avec succès. Vous pouvez maintenant cliquez sur 'Lire les Données' pour charger la liste des champs des éléments à importer");
$url = $this->admin->generateUrl('edit', ['id' => $object->getId()]) . "#tab_2";
return $this->redirect($url);
} catch (ModelManagerException $e) {
$this->handleModelManagerException($e);
$isFormValid = false;
}
}
// show an error message if the form failed validation
if (!$isFormValid) {
if (!$this->isXmlHttpRequest()) {
$this->addFlash(
'sonata_flash_error',
$this->trans(
'flash_create_error',
array('%name%' => $this->escapeHtml($this->admin->toString($object))),
'SonataAdminBundle'
)
);
}
} elseif ($this->isPreviewRequested()) {
// pick the preview template if the form was valid and preview was requested
$templateKey = 'preview';
$this->admin->getShow();
}
}
$view = $form->createView();
// set the theme for the current Admin Form
$this->get('twig')->getExtension('form')->renderer->setTheme($view, $this->admin->getFormTheme());
return $this->render($this->admin->getTemplate($templateKey), array(
'action' => 'create',
'form' => $view,
'object' => $object,
), null);
}
}
\ No newline at end of file
......@@ -8,17 +8,17 @@ use Vich\UploaderBundle\Mapping\Annotation as Vich;
use Biopen\CoreBundle\Document\AbstractFile;
abstract class ImportState
{
const Started = "started";
{
const Started = "started";
const Downloading = "downloading";
const InProgress = "in_progress";
const Completed = "completed";
const Errors = "errors";
const InProgress = "in_progress";
const Completed = "completed";
const Errors = "errors";
const Failed = "failed";
}
/**
* @MongoDB\Document
/**
* @MongoDB\Document
* @Vich\Uploadable
* Import data into GoGoCarto. the data can imported through a static file, or via API url
* The Import can be made once for all (static import) or dynamically every X days (ImportDynamic)
......@@ -33,18 +33,18 @@ class Import extends AbstractFile
/**
* @var int
* @MongoDB\Id(strategy="INCREMENT")
* @MongoDB\Id(strategy="INCREMENT")
*/
private $id;
/**
* @var string
* @var string
* @MongoDB\Field(type="string")
*/
public $sourceName;
/**
* @var string
* @var string
* Url of API to get the data
* @MongoDB\Field(type="string")
*/
......@@ -71,7 +71,7 @@ class Import extends AbstractFile
private $needToHaveOptionsOtherThanTheOnesAddedToEachElements = false;
/**
* @var string
* @var string
* @MongoDB\Field(type="string")
*/
public $fieldToCheckElementHaveBeenUpdated;
......@@ -79,7 +79,7 @@ class Import extends AbstractFile
/**
* @MongoDB\Field(type="bool")
*/
private $geocodeIfNecessary = false;
private $geocodeIfNecessary = false;
/**
* @MongoDB\ReferenceMany(targetDocument="Biopen\CoreBundle\Document\GoGoLog", cascade={"all"})
......@@ -88,7 +88,7 @@ class Import extends AbstractFile
/**
* State of the import when processing. Type of ImportState
* When processing import, this variable is being updated in realtime, so the client can follow
* When processing import, this variable is being updated in realtime, so the client can follow
* the state of the import also in realtime
* @MongoDB\Field(type="string")
*/
......@@ -100,26 +100,26 @@ class Import extends AbstractFile
*/
private $currMessage;
/**
* After importing some Data, if the user hard delete some elements, their ids will be remembered
* so next time we do not import them again
*
* @MongoDB\Field(type="collection")
*/
private $idsToIgnore = [];
/**
* After importing some Data, if the user hard delete some elements, their ids will be remembered
* so next time we do not import them again
*
* @MongoDB\Field(type="collection")
*/
private $idsToIgnore = [];
/**
* @MongoDB\Field(type="hash")
*/
private $ontologyMapping = [];
private $ontologyMapping = [];
/**
* @MongoDB\Field(type="hash")
*/
private $taxonomyMapping = [];
private $taxonomyMapping = [];
public function __construct() {
$this->logs = new \Doctrine\Common\Collections\ArrayCollection();;
}
......@@ -128,10 +128,15 @@ class Import extends AbstractFile
public function isDynamicImport() { return false; }
public function addIdToIgnore($id)
{
$this->idsToIgnore[] = $id;
}
public function addIdToIgnore($id)
{
$this->idsToIgnore[] = $id;
}
public function isCategoriesFieldMapped()
{
return in_array('categories', array_values($this->getOntologyMapping()));
}
/**
* Get id
......@@ -185,7 +190,7 @@ class Import extends AbstractFile
public function getUrl()
{
return $this->url;
}
}
/**
* Set createMissingOptions
......@@ -309,7 +314,7 @@ class Import extends AbstractFile
* @return \Doctrine\Common\Collections\Collection $logs
*/
public function getLogs()
{
{
$logs = is_array($this->logs) ? $this->logs : $this->logs->toArray();
usort( $logs, function ($a, $b) { return $b->getCreatedAt()->getTimestamp() - $a->getCreatedAt()->getTimestamp(); });
return $logs;
......@@ -359,27 +364,27 @@ class Import extends AbstractFile
return $this->currMessage;
}
/**
* Set idsToIgnore
*
* @param collection $idsToIgnore
* @return $this
*/
public function setIdsToIgnore($idsToIgnore)
{
$this->idsToIgnore = $idsToIgnore;
return $this;
}
/**
* Get idsToIgnore
*
* @return collection $idsToIgnore
*/
public function getIdsToIgnore()
{
return $this->idsToIgnore;
}
/**
* Set idsToIgnore
*
* @param collection $idsToIgnore
* @return $this
*/
public function setIdsToIgnore($idsToIgnore)
{
$this->idsToIgnore = $idsToIgnore;
return $this;
}
/**
* Get idsToIgnore
*
* @return collection $idsToIgnore
*/
public function getIdsToIgnore()
{
return $this->idsToIgnore;
}
/**
* Set needToHaveOptionsOtherThanTheOnesAddedToEachElements
......
......@@ -2,25 +2,30 @@
{% set object = form.vars.sonata_admin.admin.subject %}
{% set mapping = object.ontologyMapping %}
<table class="table">
<thead>
<tr>
<th>Attribut d'origine</th>
<th style="width:20px"></th>
<th>à transformer en</th>
</tr>
</thead>
<tbody class="table-striped">
{% for originName, mappedName in mapping %}
{% if mapping|length == 0 %}
<div class="alert alert-info">Veuillez cliquer sur le boutton "Lire les données" afin de charger les attributs des données à importer</div>
{% else %}
<table class="table">
<thead>
<tr>
<td class="original">{{originName}}</td>
<td><i class="arrow-icon fa fa-arrow-circle-right"></i></td>
<td class="mapped"><input type="text" name="ontology[{{originName}}]" class="form-control property-selector" value={{mappedName}} /></td>
<th>Attribut d'origine</th>
<th style="width:20px"></th>
<th>à transformer en</th>
</tr>
{% endfor %}
</tbody>
</thead>
<tbody class="table-striped">
{% for originName, mappedName in mapping %}
<tr>
<td class="original">{{originName}}</td>
<td><i class="arrow-icon fa fa-arrow-circle-right"></i></td>
<td class="mapped"><input type="text" name="ontology[{{originName}}]" class="form-control property-selector" value={{mappedName}} /></td>
</tr>
{% endfor %}
</tbody>
</table>
</table>
{% endif %}
<style>
.arrow-icon { font-size: 18px; }
......
......@@ -2,25 +2,30 @@
{% set object = form.vars.sonata_admin.admin.subject %}
{% set mapping = object.taxonomyMapping %}
<table class="table">
<thead>
<tr>
<th>Catégorie d'origine</th>
<th style="width:20px"></th>
<th>à transformer en</th>
</tr>
</thead>
<tbody class="table-striped">
{% for originName, mappedName in mapping %}
{% if object.isCategoriesFieldMapped %}
<table class="table">
<thead>
<tr>
<td class="original">{{originName}}</td>
<td><i class="fa fa-arrow-circle-right"></i></td>
<td class="mapped"><input type="text" name="taxonomy[{{originName}}]" class="form-control category-selector" value={{mappedName}} /></td>
<th>Catégories lues depuis le fichier d'origine</th>
<th style="width:20px"></th>
<th>Catégories du site</th>
</tr>
{% endfor %}
</tbody>
</thead>
<tbody class="table-striped">
{% for originName, mappedName in mapping %}
<tr>
<td class="original">{{originName}}</td>
<td><i class="fa fa-arrow-circle-right"></i></td>
<td class="mapped"><input type="text" name="taxonomy[{{originName}}]" class="form-control category-selector" value={{mappedName}} /></td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<div class="alert alert-info">Veuillez d'abords sélectionner le champs correspondant aux catégories dans l'onglet 'Table de correspondance des champs' puis cliquer sur 'Sauvegarder'</div>
{% endif %}
</table>
<script>
jQuery(document).ready(function() {
......
......@@ -7,9 +7,18 @@
{% if object.lastRefresh %}
<p>Dernière mise à jour : {{ object.lastRefresh|date('d/m/Y') }}</p>
{% endif %}
{% set class = "btn-primary" %}
<button type="submit" class="btn btn-success" name="submit"><i class="fa fa-save" aria-hidden="true"></i>Sauvegarder</button>
{% include '@BiopenAdmin/partials/list__action_refresh.html.twig' %}
{% include '@BiopenAdmin/partials/list__action_refresh.html.twig' with {'class': 'btn-primary'} %}
{% set class = object.ontologyMapping|length > 0 ? 'btn-default' : 'btn-primary' %}
{% if object.id != "" %}
<a href="{{ admin.generateObjectUrl('collect', object) }}" class="btn {{ class }} view_link">
<i class="fa fa-eye" aria-hidden="true"></i>
Lire les données
</a>
{% endif %}
<a href="{{ admin.generateObjectUrl('list', object) }}" class="btn btn-default view_link">
<i class="fa fa-list" aria-hidden="true"></i>Retour à la liste
</a>
......
......@@ -10,12 +10,8 @@ file that was distributed with this source code.
#}
{% if admin.isGranted('VIEW', object) and object.id != "" %}
{% if object.ontologyMapping|length == 0 %}
<a href="{{ admin.generateObjectUrl('collect', object) }}" class="btn {{ class|default('btn-sm btn-primary') }} view_link">
<i class="fa fa-eye" aria-hidden="true"></i>
Lire les données
</a>
{% else %}
{% if object.ontologyMapping|length > 0 %}
<a href="{{ admin.generateObjectUrl('refresh', object) }}" class="btn {{ class|default('btn-sm btn-primary') }} view_link">
{% if object.lastRefresh %}
<i class="fa fa-refresh" aria-hidden="true"></i>
......
......@@ -26,6 +26,16 @@ class ElementImportMappingService
protected $parentCategoryIdToCreateMissingOptions;
protected $em;
protected $coreFields = ['id', 'name', 'categories', 'streetAddress', 'addressLocality', 'postalCode', 'addressCountry', 'latitude', 'longitude', 'images', 'owner', 'source'];
protected $mappedCoreFields = [
'title' => 'name',
'taxonomy' => 'categories',
'address' => 'streetAddress',
'city' => 'addressLocatily',
'postcode' => 'postalCode',
'country' => 'addressCountry',
'lat' => 'latitude',
'long' => 'longitude', 'lng' => 'longitude'
];
public function __construct(DocumentManager $documentManager)
{
......@@ -41,12 +51,17 @@ class ElementImportMappingService
$this->collectOntology($data, $import);
$data = $this->mapOntology($data);
// remove empty row, i.e. without name
$data = array_filter($data, function($row) { return array_key_exists('name', $row); });
$data = $this->addMissingFieldsToData($data);
// $data = $this->addMissingFieldsToData($data);
if ($import->isCategoriesFieldMapped())
{
$this->collectTaxonomy($data, $import);
$data = $this->mapTaxonomy($data);
}
$this->collectTaxonomy($data, $import);
$data = $this->mapTaxonomy($data);
$this->em->persist($import);
$this->em->flush();
return $data;
......@@ -62,6 +77,7 @@ class ElementImportMappingService
if (!in_array($key, $allNewFields)) $allNewFields[] = $key;
if (!array_key_exists($key, $ontologyMapping)) {
$value = in_array($key, $this->coreFields) ? $key : "";
if (!$value && in_array($this->mappedCoreFields[$key], $this->coreFields)) $value = $this->mappedCoreFields[$key];
$ontologyMapping[$key] = $value;
}
}
......
......@@ -10,7 +10,7 @@
"address":{"streetAddress":"5 rue Georges Jacquet Grenoble"},
"createdAt":"21/12/2018 à 18:26",
"updatedAt":"17/03/2019 à 13:38",
"categories": ["vert", "tandem", "new"],
"taxonomy": ["bleu", "tandem"],
"abstract": "Ateliers bricole",
"openHoursString": "",
"openHours": null,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment