ElementImportMappingService.php 12.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
<?php

namespace Biopen\GeoDirectoryBundle\Services;

use Doctrine\ODM\MongoDB\DocumentManager;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\ProgressBar;
use Biopen\GeoDirectoryBundle\Document\Element;
use Biopen\GeoDirectoryBundle\Document\ElementStatus;
use Biopen\GeoDirectoryBundle\Document\ModerationState;
use Biopen\GeoDirectoryBundle\Document\Coordinates;
use Biopen\GeoDirectoryBundle\Document\Option;
use Biopen\GeoDirectoryBundle\Document\OptionValue;
use Biopen\GeoDirectoryBundle\Document\UserInteractionContribution;
use Biopen\GeoDirectoryBundle\Document\PostalAddress;
use Biopen\GeoDirectoryBundle\Document\ElementUrl;
use Biopen\GeoDirectoryBundle\Document\ElementImage;
use Biopen\GeoDirectoryBundle\Document\ImportState;
use Biopen\CoreBundle\Document\GoGoLogImport;
use Biopen\CoreBundle\Document\GoGoLogLevel;

class ElementImportMappingService
{
  protected $import;
25 26
  protected $createMissingOptions;
  protected $parentCategoryIdToCreateMissingOptions;
27
  protected $em;
28 29
  protected $ontologyMapping;
  protected $allNewFields;
30
  protected $coreFields = ['id', 'name', 'categories', 'streetAddress', 'addressLocality', 'postalCode', 'addressCountry', 'latitude', 'longitude', 'images', 'owner', 'source'];
Sebastian Castro's avatar
Sebastian Castro committed
31 32 33 34 35 36 37 38 39 40
  protected $mappedCoreFields = [
    'title' => 'name',
    'taxonomy' => 'categories',
    'address' => 'streetAddress',
    'city' => 'addressLocatily',
    'postcode' => 'postalCode',
    'country' => 'addressCountry',
    'lat' => 'latitude',
    'long' => 'longitude', 'lng' => 'longitude'
  ];
41

42
  public function __construct(DocumentManager $documentManager)
43
  {
44
    $this->em = $documentManager;
45 46 47 48 49
  }

  public function transform($data, $import)
  {
    $this->import = $import;
50 51 52 53
    $this->createMissingOptions = $import->getCreateMissingOptions();
    $parent = $import->getParentCategoryToCreateOptions() ?: $this->em->getRepository('BiopenGeoDirectoryBundle:Category')->findOneByIsRootCategory(true);
    $this->parentCategoryIdToCreateMissingOptions = $parent->getId();

54 55 56
    // Execute custom code (the <?php is used to have proper code highliting in text editor, we remove it before executing)
    eval(str_replace('<?php', '', $import->getCustomCode()));

57 58 59
    // elements is ofently stored nested in a data attribute
    if (array_key_exists('data', $data)) $data = $data['data'];

60
    // Fixs gogocarto ontology when importing, to simplify import/export from gogocarto to gogocarto
61
    foreach ($data as $key => $row) {
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
      if (is_array($row)) {
        if (array_key_exists('geo', $row))
        {
          $data[$key]['latitude']  = $row['geo']['latitude'];
          $data[$key]['longitude'] = $row['geo']['longitude'];
          unset($data[$key]['geo']);
        }
        if (array_key_exists('address', $row))
        {
          $address = $row['address'];

          if (gettype($address) == "string") $data[$key]['streetAddress'] = $address;
          else if ($address) {
            if (array_key_exists('streetNumber', $address))    $data[$key]['streetNumber']   =  $address['streetNumber'];
            if (array_key_exists('streetAddress', $address))   $data[$key]['streetAddress']   = $address['streetAddress'];
            if (array_key_exists('addressLocality', $address)) $data[$key]['addressLocality'] = $address['addressLocality'];
            if (array_key_exists('postalCode', $address))      $data[$key]['postalCode']      = $address['postalCode'];
            if (array_key_exists('addressCountry', $address))  $data[$key]['addressCountry']  = $address['addressCountry'];
          }
          unset($data[$key]['address']);
82
        }
83 84 85 86 87 88 89
        if (array_key_exists('categories', $row) && array_key_exists('categoriesFull', $row)) {
          $data[$key]['categories'] = $data[$key]['categoriesFull'];
          unset($data[$key]['categoriesFull']);
        }
      } else {
        // the $row is not an array, probably a string so we ignore it
        unset($data[$key]);
90 91 92
      }
    }

93
    // Ontology
94
    $this->collectOntology($data, $import);
95
    $data = $this->mapOntology($data);
96 97
    // remove empty row, i.e. without name
    $data = array_filter($data, function($row) { return array_key_exists('name', $row); });
Sebastian Castro's avatar
Sebastian Castro committed
98

99
    // Taxonomy
Sebastian Castro's avatar
Sebastian Castro committed
100 101 102 103 104
    if ($import->isCategoriesFieldMapped())
    {
      $this->collectTaxonomy($data, $import);
      $data = $this->mapTaxonomy($data);
    }
105

106 107
    $this->em->persist($import);
    $this->em->flush();
108 109 110
    return $data;
  }

111 112
  public function collectOntology($data, $import)
  {
113 114
    $this->ontologyMapping = $import->getOntologyMapping();
    $this->allNewFields = [];
115 116 117
    foreach($data as $row)
    {
      foreach ($row as $key => $value) {
118 119 120
        $this->collectKey($key);
        if ($this->isAssociativeArray($value)) {
          foreach ($value as $subkey => $subvalue) { $this->collectKey($subkey, $key); }
121 122 123
        }
      }
    }
124
    // delete no more used fields
125 126
    foreach($this->ontologyMapping as $field => $mappedField) {
      if (!in_array($field, $this->allNewFields)) unset($this->ontologyMapping[$field]);
127 128
    }

129 130 131 132 133 134 135
    $import->setOntologyMapping($this->ontologyMapping);
  }

  private function collectKey($key, $parentKey = null) {
    $keyName = $parentKey ? $parentKey . '/' . $key : $key;
    if (!in_array($keyName, $this->allNewFields)) $this->allNewFields[] = $keyName;
    if (!array_key_exists($keyName, $this->ontologyMapping)) {
136
      $value = in_array($keyName, $this->coreFields) ? $key : "";
137 138
      if (!$value && array_key_exists($key, $this->mappedCoreFields) && in_array($this->mappedCoreFields[$key], $this->coreFields))
        $value = $this->mappedCoreFields[$key];
139
      if (!$value || !in_array($value, array_values($this->ontologyMapping))) $this->ontologyMapping[$keyName] = $value;
140 141 142 143 144 145
    }
  }

  private function isAssociativeArray($a) {
    if (!is_array($a)) return false;
    foreach(array_keys($a) as $key)
146 147
      if (!is_int($key)) return true;
    return false;
148 149 150 151 152
  }

  public function collectTaxonomy($data, $import)
  {
    $taxonomyMapping = $import->getTaxonomyMapping();
153
    $allNewCategories = [];
154 155 156 157
    $this->createOptionsMappingTable();

    foreach($data as $row)
    {
158 159 160
      $categories = $row['categories'];
      $categories = is_array($categories) ? $categories : explode(',', $categories);
      foreach($categories as $category) {
161
        if (is_array($category)) $category = $category['name'];
162
        if (!in_array($category, $allNewCategories)) $allNewCategories[] = $category;
163 164
        if ($category && !array_key_exists($category, $taxonomyMapping))
        {
165 166
          $categorySlug = $this->slugify($category);
          $value = array_key_exists($categorySlug, $this->mappingTableIds) ? $this->mappingTableIds[$categorySlug]['id'] : '';
167 168 169 170

          // create option if does not exist
          if ($value == '' && $this->createMissingOptions) $value = $this->createOption($category);

171 172
          $taxonomyMapping[$category] = $value;
        }
173 174 175 176 177 178
        // create options for previously imported non mapped options
        if (array_key_exists($category, $taxonomyMapping)
            && (!$taxonomyMapping[$category] || $taxonomyMapping[$category] == '/')
            && $this->createMissingOptions) {
          $taxonomyMapping[$category] = $this->createOption($category);
        }
179 180
      }
    }
181 182 183 184
    // delete no more used categories
    foreach($taxonomyMapping as $category => $mappedCategory) {
      if (!in_array($category, $allNewCategories)) unset($taxonomyMapping[$category]);
    }
185 186 187
    $import->setTaxonomyMapping($taxonomyMapping);
  }

188 189 190 191
  private function mapOntology($data)
  {
    $mapping = $this->import->getOntologyMapping();
    foreach ($data as $key => $row) {
192 193

      // First map nested fields
194
      foreach ($mapping as $search => $replace) {
195
        $searchKeys = explode('/', $search);
196 197
        if (count($searchKeys) == 2) {
          $searchkey = $searchKeys[0]; $subkey = $searchKeys[1];
198

199
          if (!in_array($replace, ['/', '']) && isset($data[$key][$searchkey]) && !isset($data[$key][$replace])) {
200 201
            $data[$key][$replace] = $row[$searchkey][$subkey];
            unset($data[$key][$searchkey][$subkey]);
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
          }
        }
      }

      // Then remove non mapped fields
      foreach ($mapping as $search => $replace) {
        if (in_array($replace, ['/', ''])) unset($data[$key][$search]);
      }

      // Finally map non nested fields
      foreach ($mapping as $search => $replace) {
        $searchKeys = explode('/', $search);
        if (count($searchKeys) == 1) {
          $searchkey = $search;

          if (!in_array($replace, ['/', '']) && isset($data[$key][$searchkey]) && !isset($data[$key][$replace])) {
218 219 220
            $data[$key][$replace] = $row[$searchkey];
            unset($data[$key][$searchkey]);
          }
221 222 223
        }
      }

224 225 226 227
      // add streetNumber into streetAddress
      if (isset($data[$key]['streetNumber']) && isset($data[$key]['streetAddress'])) {
        $data[$key]['streetAddress'] = $data[$key]['streetNumber'] . ' ' . $data[$key]['streetAddress'];
        unset($data[$key]['streetNumber']);
228 229
      }
    }
230

231 232 233
    return $data;
  }

234
  private function mapTaxonomy($data)
235
  {
236
    $mapping = $this->import->getTaxonomyMapping();
237 238
    foreach ($data as $key => $row)
    {
239
      if (isset($data[$key]['categories'])) {
240
        if (is_string($row['categories'])) $row['categories'] = explode(',', $row['categories']);
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
        $categories = []; $categoriesIds = [];
        foreach ($row['categories'] as $category)
        {
          $val = is_array($category) ? $category['name'] : $category;
          if (array_key_exists($val, $mapping) && array_key_exists($mapping[$val], $this->mappingTableIds))
          {
            $newcat['originalValue'] = $val;
            $newcat['mappedName'] = $this->mappingTableIds[$mapping[$val]]['name'];
            $newcat['mappedId'] = $this->mappingTableIds[$mapping[$val]]['id'];
            if (isset($category['index'])) $newcat['index'] = $category['index'];
            if (isset($category['description'])) $newcat['description'] = $category['description'];
            $categories[] = $newcat;
            $categoriesIds[] = $newcat['mappedId'];
            $parentIds = $this->mappingTableIds[$mapping[$val]]['idAndParentsId'];
            foreach ($parentIds as $id) {
              if (!in_array($id, $categoriesIds)) {
                $categories[] = ['mappedId' => $id, 'info' => "Automatiquement ajoutée (category parente d'une category importée)"];
                $categoriesIds[] = $id;
              }
            }
          }
        }
        $data[$key]['categories'] = $categories;
264
      }
265
    }
266
    return $data;
267 268
  }

Sebastian Castro's avatar
Sebastian Castro committed
269 270 271 272 273 274
  private function array_flatten(array $array) {
    $return = array();
    array_walk_recursive($array, function($a) use (&$return) { $return[] = $a; });
    return $return;
  }

275 276 277 278 279 280 281 282
  private function createOptionsMappingTable($options = null)
  {
    if ($options === null) $options = $this->em->getRepository('BiopenGeoDirectoryBundle:Option')->findAll();

    foreach($options as $option)
    {
      $ids = [
        'id' => $option->getId(),
283
        'name' => $option->getName(),
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
        'idAndParentsId' => $option->getIdAndParentOptionIds()
      ];
      $this->mappingTableIds[$this->slugify($option->getNameWithParent())] = $ids;
      $this->mappingTableIds[$this->slugify($option->getName())] = $ids;
      $this->mappingTableIds[strval($option->getId())] = $ids;
      if ($option->getCustomId()) $this->mappingTableIds[$this->slugify($option->getCustomId())] = $ids;
    }
  }

  private function createOption($name)
  {
    $option = new Option();
    $option->setName($name);
    $parent = $this->em->getRepository('BiopenGeoDirectoryBundle:Category')->find($this->parentCategoryIdToCreateMissingOptions);
    $option->setParent($parent);
    $option->setUseIconForMarker(false);
    $option->setUseColorForMarker(false);
    $this->em->persist($option);
    $this->createOptionsMappingTable([$option]);
303
    return $option->getId();
304 305 306 307
  }

  private function slugify($text)
  {
308
    if (!is_string($text)) return;
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
    // replace non letter or digits by -
    $text = str_replace('é', 'e', $text);
    $text = str_replace('è', 'e', $text);
    $text = str_replace('ê', 'e', $text);
    $text = str_replace('ô', 'o', $text);
    $text = str_replace('ç', 'c', $text);
    $text = str_replace('à', 'a', $text);
    $text = str_replace('â', 'a', $text);
    $text = str_replace('î', 'i', $text);
    $text = preg_replace('~[^\pL\d]+~u', '-', $text);

    $text = iconv('utf-8', 'us-ascii//TRANSLIT', $text); // transliterate
    $text = preg_replace('~[^-\w]+~', '', $text); // remove unwanted characters
    $text = trim($text, '-'); // trim
    $text = rtrim($text, 's'); // remove final "s" for plural
    $text = preg_replace('~-+~', '-', $text); // remove duplicate -
    $text = strtolower($text); // lowercase

    if (empty($text)) return '';
    return $text;
  }
}