ElementImportMappingService.php 14.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
<?php

namespace Biopen\GeoDirectoryBundle\Services;

use Doctrine\ODM\MongoDB\DocumentManager;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\ProgressBar;
use Biopen\GeoDirectoryBundle\Document\Element;
use Biopen\GeoDirectoryBundle\Document\ElementStatus;
use Biopen\GeoDirectoryBundle\Document\ModerationState;
use Biopen\GeoDirectoryBundle\Document\Coordinates;
use Biopen\GeoDirectoryBundle\Document\Option;
use Biopen\GeoDirectoryBundle\Document\OptionValue;
use Biopen\GeoDirectoryBundle\Document\UserInteractionContribution;
use Biopen\GeoDirectoryBundle\Document\PostalAddress;
use Biopen\GeoDirectoryBundle\Document\ElementUrl;
use Biopen\GeoDirectoryBundle\Document\ElementImage;
use Biopen\GeoDirectoryBundle\Document\ImportState;
use Biopen\CoreBundle\Document\GoGoLogImport;
use Biopen\CoreBundle\Document\GoGoLogLevel;

class ElementImportMappingService
{
  protected $import;
25 26
  protected $createMissingOptions;
  protected $parentCategoryIdToCreateMissingOptions;
27
  protected $em;
28 29
  protected $ontologyMapping;
  protected $allNewFields;
30 31
  protected $existingProps;
  protected $coreFields = ['id', 'name', 'categories', 'streetAddress', 'addressLocality', 'postalCode', 'addressCountry', 'latitude', 'longitude', 'images', 'owner', 'source', 'openHours', 'email'];
32
  protected $mappedCoreFields = [
33
    'title' => 'name', 'nom' => 'name',
34
    'taxonomy' => 'categories',
35 36 37 38
    'address' => 'streetAddress',
    'city' => 'addressLocality',
    'postcode' => 'postalCode',
    'country' => 'addressCountry',
39
    'lat' => 'latitude',
40
    'long' => 'longitude', 'lng' => 'longitude', 'lon' => 'longitude'
41
  ];
42

43
  public function __construct(DocumentManager $documentManager)
44
  {
45
    $this->em = $documentManager;
46 47 48 49 50
  }

  public function transform($data, $import)
  {
    $this->import = $import;
51 52 53 54
    $this->createMissingOptions = $import->getCreateMissingOptions();
    $parent = $import->getParentCategoryToCreateOptions() ?: $this->em->getRepository('BiopenGeoDirectoryBundle:Category')->findOneByIsRootCategory(true);
    $this->parentCategoryIdToCreateMissingOptions = $parent->getId();

55 56 57
    // Execute custom code (the <?php is used to have proper code highliting in text editor, we remove it before executing)
    eval(str_replace('<?php', '', $import->getCustomCode()));

58 59
    // elements is ofently stored nested in a data attribute
    if (array_key_exists('data', $data)) $data = $data['data'];
60
    if (array_key_exists('elements', $data)) $data = $data['elements'];
61

62
    // Fixs gogocarto ontology when importing, to simplify import/export from gogocarto to gogocarto
63
    foreach ($data as $key => $row) {
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
      if (is_array($row)) {
        if (array_key_exists('geo', $row))
        {
          $data[$key]['latitude']  = $row['geo']['latitude'];
          $data[$key]['longitude'] = $row['geo']['longitude'];
          unset($data[$key]['geo']);
        }
        if (array_key_exists('address', $row))
        {
          $address = $row['address'];

          if (gettype($address) == "string") $data[$key]['streetAddress'] = $address;
          else if ($address) {
            if (array_key_exists('streetNumber', $address))    $data[$key]['streetNumber']   =  $address['streetNumber'];
            if (array_key_exists('streetAddress', $address))   $data[$key]['streetAddress']   = $address['streetAddress'];
            if (array_key_exists('addressLocality', $address)) $data[$key]['addressLocality'] = $address['addressLocality'];
            if (array_key_exists('postalCode', $address))      $data[$key]['postalCode']      = $address['postalCode'];
            if (array_key_exists('addressCountry', $address))  $data[$key]['addressCountry']  = $address['addressCountry'];
          }
          unset($data[$key]['address']);
84
        }
85 86 87 88 89 90 91
        if (array_key_exists('categories', $row) && array_key_exists('categoriesFull', $row)) {
          $data[$key]['categories'] = $data[$key]['categoriesFull'];
          unset($data[$key]['categoriesFull']);
        }
      } else {
        // the $row is not an array, probably a string so we ignore it
        unset($data[$key]);
92 93 94
      }
    }

95
    // Ontology
96
    $this->collectOntology($data, $import);
97
    $data = $this->mapOntology($data);
Sebastian Castro's avatar
Sebastian Castro committed
98 99

    if (!is_array($data)) return [];
100 101
    // remove empty row, i.e. without name
    $data = array_filter($data, function($row) { return array_key_exists('name', $row); });
102

103
    // Taxonomy
104 105 106 107 108
    if ($import->isCategoriesFieldMapped())
    {
      $this->collectTaxonomy($data, $import);
      $data = $this->mapTaxonomy($data);
    }
109

110 111
    $this->em->persist($import);
    $this->em->flush();
112 113 114
    return $data;
  }

115 116
  public function collectOntology($data, $import)
  {
117 118
    $this->ontologyMapping = $import->getOntologyMapping();
    $this->allNewFields = [];
119 120 121 122 123 124 125
    $props = $this->em->getRepository('BiopenGeoDirectoryBundle:Element')->findAllCustomProperties();
    $props = array_merge($this->coreFields, $props);
    $this->existingProps = [];
    foreach ($props as $prop) {
      $this->existingProps[str_replace('_', '', strtolower($prop))] = $prop;
    }

126 127 128
    foreach($data as $row)
    {
      foreach ($row as $key => $value) {
129
        $this->collectKey($key, null, $import);
130
        if ($this->isAssociativeArray($value) && !in_array($key, ['openHours', 'modifiedElement'])) {
131
          foreach ($value as $subkey => $subvalue) { $this->collectKey($subkey, $key, $import); }
132 133 134
        }
      }
    }
135
    // delete no more used fields
136 137
    foreach($this->ontologyMapping as $field => $mappedField) {
      if (!in_array($field, $this->allNewFields)) unset($this->ontologyMapping[$field]);
138 139
    }

140 141 142
    $import->setOntologyMapping($this->ontologyMapping);
  }

143
  private function collectKey($key, $parentKey = null, $import) {
144
    if (in_array($key, ['__initializer__', '__cloner__', '__isInitialized__'])) return;
145
    $keyName = $parentKey ? $parentKey . '/' . $key : $key;
146
    if (!$keyName || strlen($keyName) == 0) return;
147

148 149
    if (!in_array($keyName, $this->allNewFields)) $this->allNewFields[] = $keyName;
    if (!array_key_exists($keyName, $this->ontologyMapping)) {
150
      $keyLower = str_replace('_', '', strtolower($key));
151
      $value = array_key_exists($keyLower, $this->existingProps) ? $this->existingProps[$keyLower] : "";
152
      // use alternative name, like lat instead of latitude
153
      if (!$value && array_key_exists($keyLower, $this->mappedCoreFields))
154 155 156
        $value = $this->mappedCoreFields[$keyLower];
      // Asign mapping
      if (!$value || !in_array($value, array_values($this->ontologyMapping)))
157
      {
158
        $this->ontologyMapping[$keyName] = $value;
159 160
        $import->setNewOntologyToMap(true);
      }
161 162 163 164 165 166
    }
  }

  private function isAssociativeArray($a) {
    if (!is_array($a)) return false;
    foreach(array_keys($a) as $key)
167 168
      if (!is_int($key)) return true;
    return false;
169 170 171 172 173
  }

  public function collectTaxonomy($data, $import)
  {
    $taxonomyMapping = $import->getTaxonomyMapping();
174
    $allNewCategories = [];
175 176 177 178
    $this->createOptionsMappingTable();

    foreach($data as $row)
    {
179 180 181
      $categories = $row['categories'];
      $categories = is_array($categories) ? $categories : explode(',', $categories);
      foreach($categories as $category) {
182
        if (is_array($category)) $category = $category['name'];
183
        $category = ltrim(rtrim($category));
184
        if (!in_array($category, $allNewCategories)) $allNewCategories[] = $category;
185 186
        if ($category && !array_key_exists($category, $taxonomyMapping))
        {
187 188
          $categorySlug = $this->slugify($category);
          $value = array_key_exists($categorySlug, $this->mappingTableIds) ? $this->mappingTableIds[$categorySlug]['id'] : '';
189 190 191 192

          // create option if does not exist
          if ($value == '' && $this->createMissingOptions) $value = $this->createOption($category);

193
          $taxonomyMapping[$category] = [$value];
194
          $import->setNewTaxonomyToMap(true);
195
        }
196 197 198 199
        // create options for previously imported non mapped options
        if (array_key_exists($category, $taxonomyMapping)
            && (!$taxonomyMapping[$category] || $taxonomyMapping[$category] == '/')
            && $this->createMissingOptions) {
200
          $taxonomyMapping[$category] = [$this->createOption($category)];
201
        }
202 203
      }
    }
204 205 206 207
    // delete no more used categories
    foreach($taxonomyMapping as $category => $mappedCategory) {
      if (!in_array($category, $allNewCategories)) unset($taxonomyMapping[$category]);
    }
208 209 210
    $import->setTaxonomyMapping($taxonomyMapping);
  }

211 212 213 214
  private function mapOntology($data)
  {
    $mapping = $this->import->getOntologyMapping();
    foreach ($data as $key => $row) {
215 216

      // First map nested fields
217
      foreach ($mapping as $search => $replace) {
218
        $searchKeys = explode('/', $search);
219 220
        if (count($searchKeys) == 2) {
          $searchkey = $searchKeys[0]; $subkey = $searchKeys[1];
221

222
          if (!in_array($replace, ['/', '']) && isset($data[$key][$searchkey]) && isset($row[$searchkey][$subkey]) && !isset($data[$key][$replace])) {
223 224
            $data[$key][$replace] = $row[$searchkey][$subkey];
            unset($data[$key][$searchkey][$subkey]);
225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
          }
        }
      }

      // Then remove non mapped fields
      foreach ($mapping as $search => $replace) {
        if (in_array($replace, ['/', ''])) unset($data[$key][$search]);
      }

      // Finally map non nested fields
      foreach ($mapping as $search => $replace) {
        $searchKeys = explode('/', $search);
        if (count($searchKeys) == 1) {
          $searchkey = $search;

          if (!in_array($replace, ['/', '']) && isset($data[$key][$searchkey]) && !isset($data[$key][$replace])) {
241 242 243
            $data[$key][$replace] = $row[$searchkey];
            unset($data[$key][$searchkey]);
          }
244 245 246
        }
      }

247 248 249 250
      // add streetNumber into streetAddress
      if (isset($data[$key]['streetNumber']) && isset($data[$key]['streetAddress'])) {
        $data[$key]['streetAddress'] = $data[$key]['streetNumber'] . ' ' . $data[$key]['streetAddress'];
        unset($data[$key]['streetNumber']);
251
      }
Sebastian Castro's avatar
Sebastian Castro committed
252 253 254 255
      if (isset($data[$key]['fullAddress'])) {
        $data[$key]['streetAddress'] = $data[$key]['fullAddress'];
        unset($data[$key]['fullAddress']);
      }
256
    }
257

258 259 260
    return $data;
  }

261
  private function mapTaxonomy($data)
262
  {
263
    $mapping = $this->import->getTaxonomyMapping();
264 265
    foreach ($data as $key => $row)
    {
266
      if (isset($data[$key]['categories'])) {
267
        if (is_string($row['categories'])) $row['categories'] = explode(',', $row['categories']);
268 269 270 271
        $categories = []; $categoriesIds = [];
        foreach ($row['categories'] as $category)
        {
          $val = is_array($category) ? $category['name'] : $category;
272
          if (isset($mapping[$val]) && $mapping[$val])
273
          {
274 275 276 277 278 279
            foreach ($mapping[$val] as $mappedCategory) {
              if (array_key_exists($mappedCategory, $this->mappingTableIds))
              {
                $newcat['originalValue'] = $val;
                $newcat['mappedName'] = $this->mappingTableIds[$mappedCategory]['name'];
                $newcat['mappedId'] = $this->mappingTableIds[$mappedCategory]['id'];
280 281 282 283 284 285 286 287 288 289 290 291
                if (!in_array($newcat['mappedId'], $categoriesIds))
                {
                  if (isset($category['index'])) $newcat['index'] = $category['index'];
                  if (isset($category['description'])) $newcat['description'] = $category['description'];
                  $categories[] = $newcat;
                  $categoriesIds[] = $newcat['mappedId'];
                  $parentIds = $this->mappingTableIds[$mappedCategory]['idAndParentsId'];
                  foreach ($parentIds as $id) {
                    if (!in_array($id, $categoriesIds)) {
                      $categories[] = ['mappedId' => $id, 'info' => "Automatiquement ajoutée (category parente d'une category importée)"];
                      $categoriesIds[] = $id;
                    }
292 293
                  }
                }
294 295 296 297 298
              }
            }
          }
        }
        $data[$key]['categories'] = $categories;
299
      }
300
    }
301
    return $data;
302 303
  }

304 305 306 307 308 309
  private function array_flatten(array $array) {
    $return = array();
    array_walk_recursive($array, function($a) use (&$return) { $return[] = $a; });
    return $return;
  }

310 311 312 313 314 315 316 317
  private function createOptionsMappingTable($options = null)
  {
    if ($options === null) $options = $this->em->getRepository('BiopenGeoDirectoryBundle:Option')->findAll();

    foreach($options as $option)
    {
      $ids = [
        'id' => $option->getId(),
318
        'name' => $option->getName(),
319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
        'idAndParentsId' => $option->getIdAndParentOptionIds()
      ];
      $this->mappingTableIds[$this->slugify($option->getNameWithParent())] = $ids;
      $this->mappingTableIds[$this->slugify($option->getName())] = $ids;
      $this->mappingTableIds[strval($option->getId())] = $ids;
      if ($option->getCustomId()) $this->mappingTableIds[$this->slugify($option->getCustomId())] = $ids;
    }
  }

  private function createOption($name)
  {
    $option = new Option();
    $option->setName($name);
    $parent = $this->em->getRepository('BiopenGeoDirectoryBundle:Category')->find($this->parentCategoryIdToCreateMissingOptions);
    $option->setParent($parent);
    $option->setUseIconForMarker(false);
    $option->setUseColorForMarker(false);
    $this->em->persist($option);
    $this->createOptionsMappingTable([$option]);
338
    return $option->getId();
339 340 341 342
  }

  private function slugify($text)
  {
343
    if (!is_string($text)) return;
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
    // replace non letter or digits by -
    $text = str_replace('é', 'e', $text);
    $text = str_replace('è', 'e', $text);
    $text = str_replace('ê', 'e', $text);
    $text = str_replace('ô', 'o', $text);
    $text = str_replace('ç', 'c', $text);
    $text = str_replace('à', 'a', $text);
    $text = str_replace('â', 'a', $text);
    $text = str_replace('î', 'i', $text);
    $text = preg_replace('~[^\pL\d]+~u', '-', $text);

    $text = iconv('utf-8', 'us-ascii//TRANSLIT', $text); // transliterate
    $text = preg_replace('~[^-\w]+~', '', $text); // remove unwanted characters
    $text = trim($text, '-'); // trim
    $text = rtrim($text, 's'); // remove final "s" for plural
    $text = preg_replace('~-+~', '-', $text); // remove duplicate -
    $text = strtolower($text); // lowercase

    if (empty($text)) return '';
    return $text;
  }
}