Skip to content
GitLab
Commit 5ee66c70 authored by Blaise de Carné's avatar Blaise de Carné
Browse files

Merge branch 'feature/monitor'

parent 662f84d3
Pipeline #22679 failed with stage
in 12 seconds
Hide whitespace changes
Inline Side-by-side
CHANGELOG.md
View file @ 5ee66c70
......@@ -6,6 +6,14 @@ et le projet utilise [Semantic Versioning](http://semver.org/).
## [Unreleased]
## [0.2.0] - 2022-02-11
### Ajout
- Ajout de la section Anomalies
### Correction
- Explorer : le tri par commune fonctionne
## [0.1.0] - 2022-02-06
### Ajout
......
assets/react/components/Tabs.tsx
View file @ 5ee66c70
......@@ -46,7 +46,7 @@ export const Tab = (props: TabProps) => {
return (
<li className="mr-2">
{/* initialy py-4 */}
<button className={`inline-block py-2 px-4 text-sm font-medium text-center border-b-2 ${className}`} onClick={onClick}>
<button className={`inline-block py-2 px-4 font-medium text-center border-b-2 ${className}`} onClick={onClick}>
{children}
</button>
</li>
......
assets/react/dashboard/components/AnomalyEvolutionWidget.tsx
View file @ 5ee66c70
......@@ -43,12 +43,24 @@ const AnomalyEvolutionWidget = () => {
min_doc_count: 1,
};
const query = {
bool: {
filter: {
range: {
date: {
gte: "now-1M",
},
},
},
},
};
useEffect(() => {
client
.search("statistics", {
size: 0,
body: {
query: { match_all: {} },
query,
aggs: {
date: {
date_histogram,
......
assets/react/dashboard/components/KpiWidget.tsx
View file @ 5ee66c70
......@@ -31,9 +31,13 @@ const KpiWidget = () => {
const promises = [];
const query = {
range: {
date: {
gte: moment().add(-30, "days").format("YYYY-MM-DD"),
bool: {
filter: {
range: {
date: {
gte: "now-1M",
},
},
},
},
};
......
assets/react/dashboard/components/POIEvolutionWidget.tsx
View file @ 5ee66c70
......@@ -41,12 +41,24 @@ const POIEvolutionWidget = () => {
min_doc_count: 1,
};
const query = {
bool: {
filter: {
range: {
date: {
gte: "now-1M",
},
},
},
},
};
useEffect(() => {
client
.search("statistics", {
size: 0,
body: {
query: { match_all: {} },
query,
aggs: {
date: {
date_histogram,
......
assets/react/explorer/Explorer.tsx
View file @ 5ee66c70
......@@ -35,7 +35,7 @@ const Explorer = ({ url }: { url: string }) => {
<ItemDrawer uri={selectedItem} onClose={() => setSelectedItem(undefined)} />
<ReactiveBase app="datatourisme" className="h-full" url={url} theme={theme}>
<div className="flex h-full">
<PerfectScrollbar className="w-64 p-3" options={{ wheelPropagation: false }}>
<PerfectScrollbar className="w-80 p-3" options={{ wheelPropagation: false }}>
<AggregationsSidebar />
</PerfectScrollbar>
<div className="flex-1 flex flex-col h-full">
......
assets/react/explorer/components/CustomResultDataGrid.tsx
View file @ 5ee66c70
......@@ -84,6 +84,8 @@ const CustomResultDataGrid = (props: CustomResultDataGridProps) => {
},
{
key: "isLocatedAt.address.hasAddressCity.label.@fr",
sortKey:
"isLocatedAt.address.hasAddressCity.label.@fr.raw",
name: "Commune",
width: 200,
},
......
assets/react/explorer/components/drawer/ItemDrawer.tsx
View file @ 5ee66c70
......@@ -41,7 +41,7 @@ const DrawerContent = (props: any) => {
<div className="flex flex-col h-full text-black">
{image && (
<img
className="flex-shrink-0 object-cover h-48 w-full shadow-md"
className="flex-shrink-0 object-cover h-56 w-full shadow-md"
src={image}
alt={image}
onError={({ target }: any) => {
......@@ -52,15 +52,15 @@ const DrawerContent = (props: any) => {
)}
{!image && !imageError && (
<div className="flex flex-col flex-shrink-0 items-center justify-center w-full h-48 bg-gray-200">
<BiImage className="text-3xl text-gray-400 mb-2" />
<div className="flex flex-col flex-shrink-0 items-center justify-center h-56 w-full bg-gray-200">
<BiImage className="h-10 w-10 text-gray-400 mb-2" />
<div className="text-md text-gray-400">aucune image</div>
</div>
)}
{imageError && (
<div className="flex flex-col flex-shrink-0 items-center justify-center w-full h-48 bg-red-100">
<BiError className="text-3xl text-red-400 mb-1" />
<div className="flex flex-col flex-shrink-0 items-center justify-center h-56 w-full bg-red-100">
<BiError className="h-10 w-10 text-red-400 mb-1" />
<div className="text-md text-red-400">impossible de charger l'image</div>
</div>
)}
......@@ -74,7 +74,7 @@ const DrawerContent = (props: any) => {
<Tab item="overview">Aperçu</Tab>
<Tab item="raw">Données brutes</Tab>
{/* <Tab item="map">Localisation</Tab> */}
<Tab item="anomalies">Anomalies <span className={`${countAnomaly > 0 ? 'bg-red-600' : 'bg-slate-600'} leading-none text-white py-0 px-2 text-xs rounded-xl`}>{countAnomaly}</span></Tab>
<Tab item="anomalies">Anomalies <span className={`${countAnomaly > 0 ? 'bg-red-600' : 'bg-slate-600'} leading-none text-white py-0 px-2 text-sm rounded-xl`}>{countAnomaly}</span></Tab>
</TabList>
<TabPanel item="overview" children={<OverviewPanel data={data} />} />
<TabPanel item="raw" children={<RawPanel data={data} />} />
......
assets/react/explorer/components/drawer/components/DescriptionList.tsx
View file @ 5ee66c70
......@@ -22,7 +22,7 @@ export const DescriptionListItem = ({ label, children, className}: {
export const DescriptionList = ({ children }: { children: ReactNode }) => {
return (
<div className="border-t border-b border-gray-200 my-5 text-sm">
<div className="border-t border-b border-gray-200">
<dl className="mb-0">{children}</dl>
</div>
);
......
assets/react/explorer/components/drawer/panels/AnomaliesPanel.tsx
View file @ 5ee66c70
......@@ -36,16 +36,16 @@ const AnomaliesPanel = ({ data }: { data: { [key: string]: any } }) => {
<div className="flex items-center mb-2">
<BiError className="h-8 w-8 mr-2 text-gray-300" />
<div className="flex-1">
<div className="text-xs text-gray-600 flex items-center leading-none">{anomalyAnalyzerLabels[item.analyzer]}</div>
<div className="text-sm text-gray-600 flex items-center leading-none">{anomalyAnalyzerLabels[item.analyzer]}</div>
<div className="text-gray-900 font-bold text-md">{item.message}</div>
</div>
</div>
<div className="">{item.description}</div>
{item.value && <div className="mt-3 text-gray-700 leading-5 text-xs font-mono">{item.value}</div>}
{item.value && <div className="mt-3 text-gray-700 leading-5 text-sm font-mono">{item.value}</div>}
</div>
{item.path && <div className="grid grid-cols-2 -mb-1">
{item.path.map((path: any) => <div key={path} className="text-gray-600 flex items-center text-xs mb-1"><BiTargetLock className="mr-1" /> {path}</div>)}
{item.path.map((path: any) => <div key={path} className="text-gray-600 flex items-center text-sm mb-1"><BiTargetLock className="mr-1" /> {path}</div>)}
</div>}
</div>
);
......
assets/react/explorer/components/drawer/panels/OverviewPanel.tsx
View file @ 5ee66c70
......@@ -42,7 +42,7 @@ const DescriptionListIndividual = ({ entities, label, lang }: { entities: any[];
{entities.map((entity) => (
<span
key={entity.uri}
className="text-xs leading-none inline-block py-1 px-2 rounded text-slate-600 bg-slate-200 last:mr-0 mr-1"
className="text-sm leading-none inline-block py-1 px-2 rounded text-slate-600 bg-slate-200 last:mr-0 mr-1"
>
{entity.label[`@${lang}`]}
</span>
......@@ -92,19 +92,20 @@ const OverviewPanel = ({ data }: { data: { [key: string]: any } }) => {
const sameAs = extract(data, "sameAs");
return (
<div className="p-3">
<div className="text-xl mb-1">{label}</div>
{types.map((type) => (
<span
key={type}
className="text-xs leading-none inline-block py-1 px-2 rounded text-slate-600 bg-slate-200 last:mr-0 mr-1"
>
{type}
</span>
))}
<div className="my-5 text-sm">{description}</div>
<>
<div className="p-3">
<div className="text-2xl mb-1">{label}</div>
{types.map((type) => (
<span
key={type}
className="text-sm leading-none inline-block py-1 px-2 rounded text-slate-600 bg-slate-200 last:mr-0 mr-1"
>
{type}
</span>
))}
{description && <div className="mt-3">{description}</div>}
</div>
<DescriptionList>
{/* Thèmes */}
<DescriptionListIndividual entities={hasTheme} label="Thèmes" lang={lang} />
......@@ -174,7 +175,11 @@ const OverviewPanel = ({ data }: { data: { [key: string]: any } }) => {
<ItemValue children={contact.legalName} />
<ItemValue children={contact.telephone} />
<ItemValue children={contact.email} />
{contact.homepage && <a href={contact.homepage} target="_blank" rel="noreferrer">{contact.homepage}</a>}
{contact.homepage && (
<a href={contact.homepage} target="_blank" rel="noreferrer">
{contact.homepage}
</a>
)}
</div>
))}
</DescriptionListItem>
......@@ -216,7 +221,7 @@ const OverviewPanel = ({ data }: { data: { [key: string]: any } }) => {
</DescriptionListItem>
)}
</DescriptionList>
</div>
</>
);
};
......
assets/react/explorer/components/drawer/panels/RawPanel.tsx
View file @ 5ee66c70
......@@ -11,7 +11,7 @@ import JsonViewer from "react-json-view";
const RawPanel = ({ data }: { data: { [key: string]: any } }) => {
return (
<div className="p-3">
<div className="text-xs">
<div className="text-sm">
<JsonViewer src={data} enableClipboard={false} displayDataTypes={false} collapsed={2} name="poi" />
</div>
</div>
......
assets/react/monitor/Monitor.tsx 0 → 100644
View file @ 5ee66c70
/*
* This file is part of the DATAtourisme project.
* 2022
* @author Conjecto <contact@conjecto.com>
* SPDX-License-Identifier: GPL-3.0-or-later
* For the full copyright and license information, please view the LICENSE file that was distributed with this source code.
*/
import React, { useEffect, useState } from "react";
import { Sparklines, SparklinesLine } from "react-sparklines";
import { FaRegEye } from "react-icons/fa";
import ElasticsearchClient, { SearchTotalHits } from "../utils/es-client";
import { format } from "../utils/number";
import { anomalyAnalyzerLabels, anomalyAnalyzers } from "../variables";
import StatsPanel from "./components/StatsPanel";
import EvolutionLabel from "./components/EvolutionLabel";
import { gotoExplorer } from "../utils/goto";
export type MonitorContextParams = {
client: ElasticsearchClient;
};
export type Stats = {
id: string;
label: string;
count: number;
objectCount: number;
volume: number;
stats: Omit<Stats, "stats">[];
};
type HorizonCounts = {
[key: string]: {
[key: string]: {
date: string;
count: number;
};
};
};
export const MonitorContext = React.createContext<MonitorContextParams>(null);
const Monitor = ({ url }: { url: string }) => {
const client = new ElasticsearchClient(url);
const [loading, setLoading] = useState(true);
const [data, setData] = useState<Stats[]>([]);
const [weeklyEvolution, setWeeklyEvolution] = useState<{ [key: string]: number[] }>({});
const [horizonCounts, setHorizonCounts] = useState<HorizonCounts>({});
/**
* Process raw data
*/
const processStats = (data: any): Stats[] => {
const totalHits = (data.hits.total as SearchTotalHits).value;
return (data.aggregations.anomalies as any).analyzer.buckets.map((b: any) => {
const { key, doc_count, parent, message } = b;
const label = anomalyAnalyzerLabels[key];
const objectCount = parent.doc_count;
const stats = message.buckets.map((b2: any) => {
const { key, doc_count, parent } = b2;
const volume = Math.round((parent.doc_count / objectCount) * 10000) / 100;
return { id: key, label: key, count: doc_count, objectCount: parent.doc_count, volume };
});
return {
id: key,
label: label || key,
count: doc_count,
objectCount,
volume: Math.round((objectCount / totalHits) * 10000) / 100,
stats,
};
});
};
useEffect(() => {
const promises = [];
// main
promises.push(
client
.search("datatourisme", {
size: 0,
track_total_hits: true,
body: {
query: { match_all: {} },
aggs: {
anomalies: {
nested: {
path: "analyze.anomalies",
},
aggs: {
analyzer: {
terms: {
field: "analyze.anomalies.analyzer",
},
aggs: {
parent: {
reverse_nested: {},
},
// --- message
message: {
terms: {
field: "analyze.anomalies.message",
},
aggs: {
parent: {
reverse_nested: {},
},
},
},
// ---
},
},
},
},
},
},
})
.then((data) => {
setData(processStats(data));
})
);
const range = {
date: {
gte: "now-6M",
},
};
const aggs = {
anomalies: {
nested: {
path: "anomalies",
},
aggs: {
analyzer: {
terms: {
field: "anomalies.analyzer",
},
aggs: {
count: {
sum: {
field: "anomalies.objectCount",
},
},
},
},
},
},
};
// for labels
promises.push(
client
.search("statistics", {
size: 0,
body: {
query: {
bool: {
filter: [{ range }],
},
},
aggs: {
horizon: {
date_range: {
field: "date",
format: "yyyy-MM-dd",
ranges: [
{ from: "now-1d", to: "now", key: "day" },
{ from: "now-1w", to: "now-1w+1d", key: "week" },
{ from: "now-1M", to: "now-1M+1d", key: "month" },
{ from: "now-3M", to: "now-3M+1d", key: "trimester" },
{ from: "now-6M", to: "now-6M+1d", key: "semester" },
],
},
aggs,
},
},
},
})
.then((data) => {
console.log(data);
const counts: HorizonCounts = {};
data.aggregations.horizon.buckets.forEach((b: any) => {
b.anomalies.analyzer.buckets.forEach(({ key, count }: any) => {
counts[key] = counts[key] ? counts[key] : {};
counts[key][b.key] = {
date: b.from_as_string,
count: count.value,
};
});
});
setHorizonCounts(counts);
})
);
// for sparkline
promises.push(
client
.search("statistics", {
size: 0,
body: {
// elasticsearch compatible
// runtime_mappings: {
// day_of_week: {
// type: "double",
// script: {
// source: "emit(doc.date.value.dayOfWeek)"
// }
// }
// },
// query: {
// bool: {
// filter: {
// term: {
// day_of_week: 1
// }
// }
// }
// },
// opensearch compatible
query: {
bool: {
filter: [
{ range },
{
script: {
script: {
source: "doc.date.value.dayOfWeek == 1", // only monday
},
},
},
],
},
},
aggs: {
date: {
date_histogram: {
field: "date",
calendar_interval: "1w",
time_zone: "Europe/Paris",
min_doc_count: 1,
},
aggs,
},
},
},
})
.then((data) => {
const evolution: { [key: string]: number[] } = {};
data.aggregations.date.buckets
.flatMap((b: any) => b.anomalies.analyzer.buckets)
.forEach(({ key, count }: any) => {
evolution[key] = evolution[key] ? evolution[key] : [];
evolution[key].push(count.value);
});
setWeeklyEvolution(evolution);
})
);
Promise.all(promises).then(() => {
setLoading(false);
});
}, []);
return (
<MonitorContext.Provider value={{ client }}>
<section className="content monitor">
<div className="flex flex-col">
<div className="box">
<div className="box-body p-0">
<table className="table table-responsive mb-0">
<thead>
<tr>
<th>Catégorie</th>
<th>Volume</th>
<th></th>
<th>Jour</th>
<th>Semaine</th>
<th>Mois</th>
<th>Trimestre</th>
<th>Semestre</th>
<th>Historique</th>
</tr>
</thead>
<tbody>
{data.map((stats) => {
const Icon = anomalyAnalyzers[stats.id].icon;
return (
<tr key={stats.id}>
<td className="p-3 whitespace-nowrap align-middle">
<div className="flex items-start">
<Icon className="flex-shrink-0 h-8 w-8 text-gray-500" />
<div className="ml-3">
<div>{stats.label}</div>
<div className="text-gray-500">
<strong>{format(stats.count)}</strong> anomalies détectés
</div>
</div>
</div>
</td>
<td className="p-3 whitespace-nowrap align-middle">
<StatsPanel stats={stats} />
</td>
<td className="p-3 whitespace-nowrap align-middle text-sm">
<button onClick={() => gotoExplorer({ anomalyAnalyzer: [stats.id] })} className="px-2 py-1 inline-flex items-center rounded-sm bg-gray-100 hover:bg-gray-200 text-gray-800">
<FaRegEye className="mr-1" /> <div>Voir la liste</div>
</button>
</td>
<td className="p-3 whitespace-nowrap align-middle text-sm">
<EvolutionLabel
origin={horizonCounts?.[stats.id]?.day?.count}
target={stats.objectCount}
date={horizonCounts?.[stats.id]?.day?.date}
/>
</td>
<td className="p-3 whitespace-nowrap align-middle text-sm">
<EvolutionLabel
origin={horizonCounts?.[stats.id]?.week?.count}
target={stats.objectCount}
date={horizonCounts?.[stats.id]?.week?.date}
/>