pp-portfolio-classifier 0.0.1

package/src/classifier.ts

@@ -0,0 +1,242 @@
+import config from "config";
+import { filter as _filter, get } from "lodash";
+import { MorningstarAPI } from "./morningstar-api";
+import { PPSecurity } from "./types";
+import { XMLHandler } from "./xml-helper";
+
+interface StockConfig {
+  salEndpoint?: "stock/equityOverview" | "stock/companyProfile";
+  viewId?: string;
+  sourcePath?: string;
+  isSingleValue?: boolean;
+  value?: string | string[]; // For static values
+  mapping?: Record<string, string | string[]> | string;
+}
+
+interface TaxonomyConfig {
+  active: boolean;
+  name: string;
+  viewId?: string;
+  // Fund config
+  sourcePath: string;
+  keyField: string;
+  valueField: string;
+  filter?: Record<string, any>;
+  mapping: Record<string, string | string[]> | string;
+  // Stock config
+  stockConfig?: StockConfig;
+}
+
+export class Classifier {
+  private xmlHandler: XMLHandler;
+  private api: MorningstarAPI;
+  private taxonomiesConfig: Record<string, TaxonomyConfig>;
+  private globalMappings: Record<string, Record<string, string | string[]>>;
+
+  constructor(xmlHandler: XMLHandler, api: MorningstarAPI) {
+    this.xmlHandler = xmlHandler;
+    this.api = api;
+    this.taxonomiesConfig = config.get("taxonomies");
+    this.globalMappings = config.has("mappings") ? config.get("mappings") : {};
+  }
+
+  private getMapping(
+    mappingOrKey: Record<string, string | string[]> | string | undefined,
+  ): Record<string, string | string[]> {
+    if (!mappingOrKey) return {};
+    if (typeof mappingOrKey === "string") {
+      if (this.globalMappings[mappingOrKey]) {
+        return this.globalMappings[mappingOrKey];
+      }
+      console.warn(`    [Config] Mapping key '${mappingOrKey}' not found in global mappings.`);
+      return {};
+    }
+    return mappingOrKey;
+  }
+
+  public async classifySecurity(security: PPSecurity) {
+    if (!security.isin && !security.isinOverride) return;
+    if (security.ignoreTaxonomies === true) return;
+
+    const securityInfo = await this.api.getSecurityData(security.isinOverride || security.isin!);
+    if (!securityInfo) {
+      console.log(`  > No data found for ${security.name}.`);
+      return;
+    }
+
+    console.log(`  > Data retrieved for ${security.name}. Type: ${securityInfo.type}. Processing taxonomies...`);
+
+    if (securityInfo.type === "Stock") {
+      await this.classifyStock(security, securityInfo.secid, securityInfo.data);
+    } else if (securityInfo.type === "Fund") {
+      await this.classifyFund(security, securityInfo.data);
+    } else {
+      console.error(`  > Unknown security type: ${securityInfo.type}! Skipping...`);
+    }
+  }
+
+  private async classifyFund(security: PPSecurity, data: any) {
+    for (const [taxonomyId, taxConfig] of Object.entries(this.taxonomiesConfig)) {
+      if (!taxConfig.active) continue;
+      if (
+        security.ignoreTaxonomies !== undefined &&
+        (security.ignoreTaxonomies === true || (security.ignoreTaxonomies as string[]).includes(taxonomyId))
+      )
+        continue;
+      console.log(`    > ${taxonomyId}`);
+
+      // 0. Fetch additional data if needed
+      if (taxConfig.viewId) {
+        const additionalData = await this.api.getSecurityData(
+          security.isinOverride || security.isin!,
+          taxConfig.viewId,
+        );
+        if (additionalData) {
+          Object.assign(data, additionalData.data);
+        }
+      }
+
+      // 1. Extract Data
+      const sourceData = get(data, taxConfig.sourcePath);
+      if (!sourceData) {
+        console.warn(`    [${taxonomyId}] No data found at path '${taxConfig.sourcePath}'`);
+        continue;
+      }
+      if (!Array.isArray(sourceData)) {
+        console.warn(`    [${taxonomyId}] Data at '${taxConfig.sourcePath}' is not an array.`);
+        continue;
+      }
+
+      // 2. Filter Data
+      // console.log(`    [${taxonomyId}] Filtering ${sourceData.length} items...`);
+      let filteredData = sourceData;
+      if (taxConfig.filter) {
+        filteredData = _filter(sourceData, taxConfig.filter);
+      }
+
+      if (filteredData.length === 0) {
+        console.log(`    [${taxonomyId}] No items match the filter.`);
+        continue;
+      }
+
+      // 2.1 Extract BreakdownValues (Flattening)
+      // The data we want is usually nested in a 'BreakdownValues' array inside the filtered items
+      let itemsToProcess: any[] = [];
+      for (const group of filteredData) {
+        if (group.BreakdownValues && Array.isArray(group.BreakdownValues)) {
+          itemsToProcess.push(...group.BreakdownValues);
+        } else {
+          itemsToProcess.push(group);
+        }
+      }
+
+      // 3. Map and Assign
+      // console.log(`    [${taxonomyId}] Assigning ${itemsToProcess.length} items...`);
+      const assignments: { path: string[]; weight: number }[] = [];
+      const mapping = this.getMapping(taxConfig.mapping);
+      for (const item of itemsToProcess) {
+        const key = item[taxConfig.keyField];
+        const value = parseFloat(item[taxConfig.valueField]);
+
+        if (key && value >= 0.01) {
+          if (!Object.keys(mapping).length) {
+            assignments.push({ path: [key], weight: value * 100 });
+          } else if (key in mapping) {
+            const targetClass = mapping[key];
+            if (targetClass) {
+              const path = Array.isArray(targetClass) ? targetClass : [targetClass];
+              assignments.push({ path, weight: value * 100 });
+              console.log(`    [${taxonomyId}] Mapped '${key}' to '${path.join(" > ")}' (${value.toFixed(2)}%)`);
+            }
+          } else {
+            console.log(`    [${taxonomyId}] Unmapped key: '${key}' (Value: ${value})`);
+          }
+        }
+      }
+      this.xmlHandler.updateSecurityAssignments(taxConfig.name || taxonomyId, security.uuid, assignments);
+    }
+  }
+
+  private async classifyStock(security: PPSecurity, secid: string | null, data: any) {
+    if (!secid && !data) {
+      console.warn(`  > Cannot classify stock ${security.name} without data.`);
+      return;
+    }
+
+    for (const [taxonomyId, taxConfig] of Object.entries(this.taxonomiesConfig)) {
+      if (!taxConfig.active) continue;
+      if (
+        security.ignoreTaxonomies !== undefined &&
+        (security.ignoreTaxonomies === true || (security.ignoreTaxonomies as string[]).includes(taxonomyId))
+      )
+        continue;
+      if (!taxConfig.stockConfig) continue;
+      console.log(`    > ${taxonomyId}`);
+
+      const stockConfig = taxConfig.stockConfig;
+
+      if (stockConfig.isSingleValue && stockConfig.value) {
+        // Case 1: Static value (e.g., Asset Type is always 'Stocks')
+        const path = Array.isArray(stockConfig.value) ? stockConfig.value : [stockConfig.value];
+        this.xmlHandler.updateSecurityAssignments(taxConfig.name || taxonomyId, security.uuid, [
+          { path, weight: 10000 },
+        ]); // 100% weight
+      } else if (stockConfig.sourcePath) {
+        // Case 2: Value from Data (SAL or Initial)
+        let key: any;
+
+        if (stockConfig.salEndpoint) {
+          if (!secid) {
+            console.warn(`    [${taxonomyId}] Cannot fetch SAL data without secid.`);
+            continue;
+          }
+          const salData = await this.api.getSalData(secid, stockConfig.salEndpoint);
+          if (!salData) {
+            console.warn(`    [${taxonomyId}] No SAL data retrieved.`);
+            continue;
+          }
+          key = get(salData, stockConfig.sourcePath);
+        } else if (stockConfig.viewId) {
+          const id = security.isinOverride || security.isin;
+          if (!id) {
+            console.warn(`    [${taxonomyId}] Cannot fetch view data without ID.`);
+            continue;
+          }
+
+          const viewResult = await this.api.getSecurityData(id, stockConfig.viewId);
+          // console.debug(security, viewResult);
+          if (viewResult && viewResult.data) {
+            key = get(viewResult.data, stockConfig.sourcePath);
+            // console.debug(security.name, viewResult.data.QuantitativeFairValue, stockConfig.sourcePath, key);
+          } else {
+            console.warn(`    [${taxonomyId}] No data retrieved for view '${stockConfig.viewId}'.`);
+            continue;
+          }
+        } else {
+          // Use data from initial request
+          key = get(data, stockConfig.sourcePath);
+        }
+
+        const mapping = this.getMapping(stockConfig.mapping);
+        if (key) {
+          // console.debug(security.name, key, mapping[key]);
+          if (!Object.keys(mapping).length) {
+            this.xmlHandler.updateSecurityAssignments(taxConfig.name || taxonomyId, security.uuid, [
+              { path: [key], weight: 10000 },
+            ]); // 100% weight
+          } else if (key in mapping) {
+            const targetClass = mapping[key];
+            if (targetClass) {
+              const path = Array.isArray(targetClass) ? targetClass : [targetClass];
+              this.xmlHandler.updateSecurityAssignments(taxConfig.name || taxonomyId, security.uuid, [
+                { path, weight: 10000 },
+              ]); // 100% weight
+            }
+          } else {
+            console.warn(`    [${taxonomyId}] Unmapped stock value '${key}'.`);
+          }
+        }
+      }
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,64 @@
+import * as path from "path";
+import { Classifier } from "./classifier";
+import { MorningstarAPI } from "./morningstar-api";
+import { XMLHandler } from "./xml-helper";
+
+async function main() {
+  const inputFile = process.argv[2];
+
+  if (!inputFile) {
+    console.error("Usage: npm start <input_file.xml> [output_file.xml]");
+    process.exit(1);
+  }
+
+  const outputFile =
+    process.argv[3] ||
+    path.join(path.dirname(inputFile), path.basename(inputFile, path.extname(inputFile)) + ".categorized.xml");
+
+  // 1. Init modules
+  const xmlHandler = new XMLHandler();
+  const api = new MorningstarAPI();
+  const classifier = new Classifier(xmlHandler, api);
+
+  // 2. Load XML
+  try {
+    xmlHandler.load(inputFile);
+  } catch (e) {
+    console.error("Error reading XML:", e);
+    process.exit(1);
+  }
+
+  // 3. Get Securities
+  const securities = xmlHandler.getSecurities();
+  console.log(`Found ${securities.length} securities.`);
+
+  // 4. Process loop
+  let processedCount = 0;
+
+  for (const sec of securities) {
+    // console.log("Processing", sec);
+    if (sec.isRetired) continue;
+    if (!sec.isin && !sec.isinOverride) {
+      console.log(`Skipping ${sec.name} (no ISIN).`);
+      continue;
+    }
+
+    console.log(`Processing ${sec.name} (${sec.isin})...`);
+
+    try {
+      await classifier.classifySecurity(sec);
+    } catch (err) {
+      console.error(`  Error processing ${sec.isin}:`, err);
+    }
+
+    processedCount++;
+    // Petit délai pour être gentil avec l'API
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+
+  // 5. Save XML (Test de compatibilité: on sauvegarde sans modif pour voir si PP l'ouvre)
+  xmlHandler.save(outputFile);
+  console.log("Done. Try opening the output file in Portfolio Performance.");
+}
+
+main();

package/src/morningstar-api.ts

@@ -0,0 +1,155 @@
+import axios, { AxiosError } from "axios";
+import config from "config";
+
+/*
+  Useful Morningstar Postman workspace for exploring the API
+  https://www.postman.com/dynamic-services-morningstar-com/morningstar-direct-web-services/request/pnun7vq/investment-details-mutual-fund-snapshot-view
+*/
+
+// Define a type for the security data response
+interface SecurityDataResponse {
+  type: "Fund" | "Stock" | "Unknown";
+  data: any;
+  secid: string | null;
+}
+
+export class MorningstarAPI {
+  private bearerToken: string = "";
+  private domain: string;
+  private baseUrl: string;
+  private salBaseUrl: string;
+  private viewId: string;
+
+  constructor() {
+    this.domain = config.get("morningstar.domain");
+    this.baseUrl = config.get("morningstar.baseUrl");
+    this.salBaseUrl = config.get("morningstar.salBaseUrl");
+    this.viewId = config.get("morningstar.viewId");
+  }
+
+  private async getBearerToken(): Promise<string> {
+    if (this.bearerToken) return this.bearerToken;
+
+    console.log("Fetching Morningstar Bearer Token...");
+    const url = `https://www.morningstar.${this.domain}/Common/funds/snapshot/PortfolioSAL.aspx`;
+
+    try {
+      const response = await axios.get(url);
+      const tokenRegex = /const maasToken \=\s\"(.+)\"/;
+      const match = response.data.match(tokenRegex);
+
+      if (match && match[1]) {
+        this.bearerToken = match[1];
+        // console.debug("Got Bearer Token:", this.bearerToken);
+        return this.bearerToken;
+      } else {
+        throw new Error("Token regex failed");
+      }
+    } catch (error) {
+      console.error("Failed to get Bearer Token", error);
+      throw error;
+    }
+  }
+
+  /**
+   * Récupère des données complémentaires via une vue spécifique (ex: 'snapshot').
+   * Peut être utilisé avec un ISIN ou un SecId.
+   */
+  async getSecurityData(id: string, viewId?: string, idType: "ISIN" | "SecId" = "ISIN"): Promise<any> {
+    const token = await this.getBearerToken();
+
+    const url = `${this.baseUrl}/securities/${id}`;
+    const params = {
+      idtype: idType,
+      viewid: viewId || this.viewId,
+      currencyId: "EUR",
+      responseViewFormat: "json",
+      languageId: "en-UK",
+    };
+    const headers = { Authorization: `Bearer ${token}`, accept: "*/*" };
+
+    try {
+      const response = await axios.get(url, { params, headers });
+      if (response.data && response.data.length > 0) {
+        const securityInfo = response.data[0];
+        // if (isin == "NL0011585146") console.debug(isin, securityInfo);
+        return {
+          type: securityInfo.Type as "Fund" | "Stock",
+          data: securityInfo,
+          secid: securityInfo.Id || id, // Fallback to ISIN if SecId is missing (SAL API often accepts ISIN)
+        };
+      }
+      // If response is empty (200 OK but no data), try fallback
+      console.log(`  > ecint API returned empty for ${id}, trying website search fallback...`);
+      return this.findSecidFromWebsite(id);
+    } catch (error) {
+      if ((error as AxiosError).response?.status === 401) {
+        // This can happen for stocks not covered by this endpoint. Try the fallback.
+        console.log(`  > ecint API failed for ${id}, trying website search fallback...`);
+        return this.findSecidFromWebsite(id);
+      }
+      console.warn(`  Warning: Could not fetch data for ${id} from ecint API. Error: ${(error as Error).message}`);
+      return null;
+    }
+  }
+
+  /**
+   * Récupère des données détaillées pour les actions via l'API SAL.
+   */
+  async getSalData(secid: string, endpoint: "stock/equityOverview" | "stock/companyProfile"): Promise<any> {
+    const token = await this.getBearerToken();
+
+    let url = `${this.salBaseUrl}/${endpoint}`;
+    let params: Record<string, string> = {
+      languageId: "en-UK",
+      locale: "en",
+    };
+    const headers = { Authorization: `Bearer ${token}`, accept: "*/*" };
+
+    if (endpoint === "stock/equityOverview") {
+      url += `/${secid}/data`;
+      params.version = "4.65.0";
+    } else {
+      // companyProfile
+      url += `/${secid}`;
+    }
+
+    try {
+      const response = await axios.get(url, { params, headers });
+      return response.data;
+    } catch (error) {
+      console.warn(`  Warning: Could not fetch SAL data for secid ${secid} from endpoint ${endpoint}.`);
+      console.debug(url, params);
+      return null;
+    }
+  }
+
+  /**
+   * Fallback: Trouve le SecId d'un titre en cherchant sur le site de Morningstar.
+   * C'est utile pour les actions qui ne sont pas dans l'API ecint.
+   */
+  private async findSecidFromWebsite(isin: string): Promise<SecurityDataResponse | null> {
+    const url = `https://global.morningstar.com/api/v1/${this.domain}/search/securities`;
+    const params = { query: `((isin ~= "${isin}"))` };
+    const headers = { "user-agent": "Mozilla/5.0" };
+
+    try {
+      const response = await axios.get(url, { params, headers });
+      const results = response.data?.results;
+      if (results && results.length > 0) {
+        const secid = results[0].securityID;
+        const universe = results[0].universe;
+        const type = universe === "EQ" ? "Stock" : "Unknown";
+
+        if (type === "Stock") {
+          console.log(`  > Found secid '${secid}' for stock ${isin} via fallback.`);
+          return { type: "Stock", data: null, secid: secid };
+        }
+      }
+      return null;
+    } catch (error) {
+      console.warn(`  Warning: Fallback search for ${isin} failed.`);
+      return null;
+    }
+  }
+}

package/src/types.ts

@@ -0,0 +1,28 @@
+export interface PPSecurity {
+  uuid: string;
+  name: string;
+  isin?: string;
+  tickerSymbol?: string;
+  note?: string;
+  isRetired?: string; // Souvent "true" ou "false" en string dans le XML
+  isinOverride?: string;
+  ignoreTaxonomies?: string[] | boolean;
+}
+
+export interface PPTaxonomy {
+  id: string;
+  name: string;
+  root: {
+    id: string;
+    name: string;
+    children?: any;
+    assignments?: any;
+  };
+}
+
+export interface MorningstarData {
+  isin: string;
+  name: string;
+  type: "Fund" | "Stock" | "Unknown";
+  data: any; // Le payload JSON complet de l'API
+}