Commit f5cde48c authored by Sebastian Castro's avatar Sebastian Castro

Element import taxonomy mapping

parent becfd621
......@@ -26,6 +26,9 @@ class ImportDynamicAdmin extends ImportAbstractAdmin
$repo = $dm->getRepository('BiopenGeoDirectoryBundle:Element');
$formProperties = json_encode($repo->findFormProperties());
$elementProperties = json_encode($repo->findDataCustomProperties());
$optionsList = $dm->getRepository('BiopenGeoDirectoryBundle:Option')->createQueryBuilder()
->select('name')->hydrate(false)->getQuery()->execute()->toArray();
$optionsList = json_encode($optionsList);
$formMapper
->tab('Général')
......@@ -59,7 +62,7 @@ class ImportDynamicAdmin extends ImportAbstractAdmin
->end()
->tab('Table de correspondance des catégories')
->with('Faites correspondre les catégories')
->add('taxonomyMapping', 'hidden', array('attr' => ['class' => 'gogo-mapping-taxonomy']))
->add('taxonomyMapping', 'hidden', array('attr' => ['class' => 'gogo-mapping-taxonomy', 'data-options' => $optionsList]))
->end()
->end()
->tab('Aide')
......
......@@ -29,9 +29,9 @@ class ImportDynamicAdminController extends Controller
$em->persist($object);
$em->flush();
$this->get('biopen.async')->callCommand('app:elements:importSource', [$object->getId()]);
// $this->get('biopen.async')->callCommand('app:elements:importSource', [$object->getId()]);
// $result = $this->get('biopen.element_import')->importJson($object);
$result = $this->get('biopen.element_import')->importJson($object);
$redirectionUrl = $this->admin->generateUrl('edit', ['id' => $object->getId()]);
$stateUrl = $this->generateUrl('biopen_import_state', ['id' => $object->getId()]);
......@@ -74,8 +74,8 @@ class ImportDynamicAdminController extends Controller
try {
// ----- CUSTOM -------
$object->setOntologyMapping($request->get('ontology'));
$object->setTaxonomyMapping($request->get('taxonomy'));
// ---- END CUSTOM ------
......
......@@ -49,7 +49,7 @@ services:
biopen.element_json_generator:
class: Biopen\GeoDirectoryBundle\EventListener\ElementJsonGenerator
tags:
- { name: doctrine_mongodb.odm.event_listener, event: preFlush }
- { name: doctrine_mongodb.odm.event_listener, event: preFlush }
biopen.db_integrity:
class: Biopen\GeoDirectoryBundle\EventListener\DatabaseIntegrityWatcher
......@@ -60,21 +60,21 @@ services:
biopen.element_import:
class: Biopen\GeoDirectoryBundle\Services\ElementImportService
arguments:
arguments:
- "@doctrine.odm.mongoDB.document_manager"
- '@biopen.element_import_one'
- "@biopen.element_import_mapping"
biopen.element_import_one:
class: Biopen\GeoDirectoryBundle\Services\ElementImportOneService
arguments:
arguments:
- "@doctrine.odm.mongoDB.document_manager"
- '@bazinga_geocoder.geocoder'
- "@biopen.user_interaction_service"
biopen.element_import_mapping:
class: Biopen\GeoDirectoryBundle\Services\ElementImportMappingService
arguments: []
arguments: ["@doctrine.odm.mongoDB.document_manager"]
biopen.gogocartojs_service:
class: Biopen\GeoDirectoryBundle\Services\GoGoCartoJsService
......
......@@ -5,7 +5,7 @@
<table class="table">
<thead>
<tr>
<th>Attribute d'origine</th>
<th>Attribut d'origine</th>
<th style="width:20px"></th>
<th>à transformer en</th>
</tr>
......@@ -79,8 +79,6 @@
}
{% endfor %}
console.log(allProperties, otherData);
options = [{ text: "Attributs du formulaire", children: formData}];
if (otherData.length > 0) options.push({text: "Autres attributs", children: otherData});
options.push({ text: "Attributs importés", children: importedData});
......
<input type="hidden" id="{{ form.vars.id }}" name="{{ form.vars.full_name }}"/>
{% set object = form.vars.sonata_admin.admin.subject %}
{% set mapping = object.ontologyMapping %}
{% set mapping = object.taxonomyMapping %}
<table class="table">
<thead>
<tr>
<th>Attribute d'origine</th>
<th>Catégorie d'origine</th>
<th style="width:20px"></th>
<th>à transformer en</th>
</tr>
......@@ -14,10 +14,28 @@
{% for originName, mappedName in mapping %}
<tr>
<td class="original">{{originName}}</td>
<td><i class="arrow-icon fa fa-arrow-circle-right"></i></td>
<td class="mapped"><input type="text" name="ontology[{{originName}}]" class="form-control property-selector" value={{mappedName}} /></td>
<td><i class="fa fa-arrow-circle-right"></i></td>
<td class="mapped"><input type="text" name="taxonomy[{{originName}}]" class="form-control category-selector" value={{mappedName}} /></td>
</tr>
{% endfor %}
</tbody>
</table>
\ No newline at end of file
</table>
<script>
jQuery(document).ready(function() {
var optionsList = JSON.parse('{{ form.vars.attr["data-options"]|raw }}');
var formData = []
for(var key in optionsList) {
var option = optionsList[key];
formData.push({id: option['_id'], text: option.name});
}
$(".category-selector").select2({
data: formData
});
});
</script>
\ No newline at end of file
......@@ -22,42 +22,82 @@ use Biopen\CoreBundle\Document\GoGoLogLevel;
class ElementImportMappingService
{
protected $import;
protected $em;
protected $coreFields = ['id', 'name', 'categories', 'streetAddress', 'addressLocality', 'postalCode', 'addressCountry', 'latitude', 'longitude', 'images', 'owner', 'source'];
public function collectData($data, $import)
public function __construct(DocumentManager $documentManager)
{
$mapping = $import->getOntologyMapping();
foreach($data as $row)
foreach ($row as $key => $value) {
if (!array_key_exists($key, $mapping)) {
$value = in_array($key, $this->coreFields) ? $key : "";
$mapping[$key] = $value;
}
}
$import->setOntologyMapping($mapping);
$this->em = $documentManager;
}
public function transform($data, $import)
{
$this->import = $import;
$this->collectOntology($data, $import);
$data = $this->mapOntology($data);
// remove empty row, i.e. without name
$data = array_filter($data, function($row) { return array_key_exists('name', $row); });
$data = $this->addMissingFieldsToData($data);
dump($data);
$this->collectTaxonomy($data, $import);
$data = $this->mapTaxonomy($data);
dump($data);
$this->em->persist($import);
$this->em->flush();
return $data;
}
public function collectOntology($data, $import)
{
$ontologyMapping = $import->getOntologyMapping();
foreach($data as $row)
{
foreach ($row as $key => $value) {
if (!array_key_exists($key, $ontologyMapping)) {
$value = in_array($key, $this->coreFields) ? $key : "";
$ontologyMapping[$key] = $value;
}
}
}
$import->setOntologyMapping($ontologyMapping);
}
public function collectTaxonomy($data, $import)
{
$taxonomyMapping = $import->getTaxonomyMapping();
$this->createOptionsMappingTable();
foreach($data as $row)
{
$taxonomy = $row['categories'];
$taxonomy = is_array($taxonomy) ? $taxonomy : explode(',', $taxonomy);
foreach($taxonomy as $category) {
if ($category && !array_key_exists($category, $taxonomyMapping)) {
$categorySlug = $this->slugify($category);
$value = array_key_exists($categorySlug, $this->mappingTableIds) ? $this->mappingTableIds[$categorySlug]['id'] : '';
$taxonomyMapping[$category] = $value;
}
}
}
$import->setTaxonomyMapping($taxonomyMapping);
}
private function mapOntology($data)
{
$mapping = $this->import->getOntologyMapping();
foreach ($data as $key => $row) {
foreach ($mapping as $search => $replace) {
if ($replace != '/' && isset($row[$search]) && !isset($row[$replace])) {
if ($replace == '/' || $replace == '') {
unset($data[$key][$search]);
}
else if (isset($row[$search]) && !isset($row[$replace])) {
$data[$key][$replace] = $data[$key][$search];
unset($data[$key][$search]);
}
}
}
return $data;
}
......@@ -72,14 +112,17 @@ class ElementImportMappingService
return $data;
}
private function mapcategories($data)
private function mapTaxonomy($data)
{
$mapping = $this->import->getcategoriesMapping();
$mapping = $this->import->getTaxonomyMapping();
foreach ($data as $key => $row)
{
if (is_string($row['categories'])) $row['categories'] = explode(',', $row['categories']);
$row['categories'] = array_map(function($el) use ($mapping) { return $mapping[$el]; }, $row['categories']);
$data[$key]['categories'] = array_map(function($el) use ($mapping) {
return array_key_exists($el, $mapping) ? $mapping[$el] : '';
}, $row['categories']);
}
return $data;
}
private function createOptionsMappingTable($options = null)
......@@ -99,7 +142,7 @@ class ElementImportMappingService
}
}
public function autoMapcategories()
public function automapTaxonomy()
{
foreach($options as $optionName)
{
......@@ -133,6 +176,7 @@ class ElementImportMappingService
private function slugify($text)
{
if (!is_string($text)) return;
// replace non letter or digits by -
$text = str_replace('é', 'e', $text);
$text = str_replace('è', 'e', $text);
......
......@@ -206,6 +206,10 @@ class ElementImportOneService
$optionsIdAdded = [];
$options = $row['categories'];
foreach ($options as $optionId) {
$this->addOptionValue($element, $optionId);
}
if ($import->getNeedToHaveOptionsOtherThanTheOnesAddedToEachElements()) {
// checking option number before adding optionIdsToAddToEachElement
if (count($element->getOptionValues()) == 0) $element->setModerationState(ModerationState::NoOptionProvided);
......@@ -221,6 +225,7 @@ class ElementImportOneService
private function addOptionValue($element, $id)
{
if (!$id) return;
$optionValue = new OptionValue();
$optionValue->setOptionId($id);
$optionValue->setIndex(0);
......
......@@ -126,9 +126,7 @@ class ElementImportService
public function collectData($import)
{
$data = $import->getUrl() ?$this->importJson($import, true) : $this->importCsv($import, true);
$this->mappingService->collectData($data, $import);
$this->em->persist($import);
$this->em->flush();
$this->mappingService->transform($data, $import);
}
public function importData($data, $import)
......@@ -137,12 +135,8 @@ class ElementImportService
// Define the size of record, the frequency for persisting the data and the current index of records
$size = count($data); $batchSize = 100; $i = 0;
// still collect data on each import because the list of fields and categories might change
$this->mappingService->collectData($data, $import);
// do the mapping
$data = $this->mappingService->transform($data, $import);
// remove empty row, i.e. without name
$data = array_filter($data, function($row) { return $row['name']; });
if ($import->isDynamicImport())
{
......
......@@ -10,7 +10,7 @@
"address":{"streetAddress":"5 rue Georges Jacquet Grenoble"},
"createdAt":"21/12/2018 à 18:26",
"updatedAt":"17/03/2019 à 13:38",
"categories": [89],
"categories": ["vert", "tandem", "Totoé"],
"abstract": "Ateliers bricole",
"openHoursString": "",
"openHours": null,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment