feat: enhance cleaned PII and custom regexes

package.json

@@ -55,6 +55,7 @@
     "cron-validator": "^1.3.1",
     "cronstrue": "^2.26.0",
     "crypto-js": "^4.1.1",
+    "data-guardian": "^1.1.3",
     "date-fns": "^2.29.3",
     "dompurify": "^3.0.6",
     "emojilib": "^3.0.10",
@@ -70,7 +71,6 @@
     "libphonenumber-js": "^1.10.28",
     "lodash": "^4.17.21",
     "marked": "^10.0.0",
-    "mask-sensitive-data": "^0.11.5",
     "mathjs": "^11.9.1",
     "mime-types": "^2.1.35",
     "monaco-editor": "^0.43.0",

pnpm-lock.yaml

@@ -65,6 +65,9 @@ dependencies:
   crypto-js:
     specifier: ^4.1.1
     version: 4.1.1
+  data-guardian:
+    specifier: ^1.1.3
+    version: 1.1.3
   date-fns:
     specifier: ^2.29.3
     version: 2.29.3
@@ -110,9 +113,6 @@ dependencies:
   marked:
     specifier: ^10.0.0
     version: 10.0.0
-  mask-sensitive-data:
-    specifier: ^0.11.5
-    version: 0.11.5
   mathjs:
     specifier: ^11.9.1
     version: 11.9.1
@@ -4722,6 +4722,11 @@ packages:
     resolution: {integrity: sha512-4FbVrHDwfOASx7uQVxeiCTo7ggSdYZbqs8lH+WU6ViypPlDbe9y6IP5VVUDQBv9DcnyaiPT5XT0UWHgJ64zLeQ==}
     dev: false
 
+  /data-guardian@1.1.3:
+    resolution: {integrity: sha512-8M9KHTopFp3MXo7NelcmF+8L77S5jeDy3Uv4RWuYIQSaBZTawvY6W3ad/eLe8db2FcLPoC44DHJb4zWNcpoueQ==}
+    engines: {node: '>=12'}
+    dev: false
+
   /data-uri-to-buffer@4.0.1:
     resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
     engines: {node: '>= 12'}
@@ -6802,10 +6807,6 @@ packages:
     resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==}
     dev: false
 
-  /map-or-similar@1.5.0:
-    resolution: {integrity: sha512-0aF7ZmVon1igznGI4VS30yugpduQW3y3GkcgGJOp7d8x8QrizhigUxjI/m2UojsXXto+jLAH3KSz+xOJTiORjg==}
-    dev: false
-
   /map-stream@0.1.0:
     resolution: {integrity: sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==}
     dev: true
@@ -6826,12 +6827,6 @@ packages:
     hasBin: true
     dev: false
 
-  /mask-sensitive-data@0.11.5:
-    resolution: {integrity: sha512-NBl/OC0kkKQaF5uo7dNnnNihidZM0TBCpxnKAHzTTmBnFPaWX9sZ2DHXOpnWRdrltgeTryTnrQ8pOutKIhkMgw==}
-    dependencies:
-      telejson: 7.2.0
-    dev: false
-
   /mathjs@11.9.1:
     resolution: {integrity: sha512-VdB9ELZ6Dwda13j5d1eCBETmPO8m9qIJETUdfZmTA9cPnXUiIk7UuoAmvxFqtfAe32XYuRugAec2Ndv0/RfRhg==}
     engines: {node: '>= 14'}
@@ -6874,12 +6869,6 @@ packages:
   /mdurl@1.0.1:
     resolution: {integrity: sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==}
 
-  /memoizerific@1.11.3:
-    resolution: {integrity: sha512-/EuHYwAPdLtXwAwSZkh/Gutery6pD2KYd44oQLhAvQp/50mpyduZh8Q7PYHXTCJ+wuXxt7oij2LXyIJOOYFPog==}
-    dependencies:
-      map-or-similar: 1.5.0
-    dev: false
-
   /merge-stream@2.0.0:
     resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==}
     dev: true
@@ -8392,12 +8381,6 @@ packages:
       tslib: 2.5.0
     dev: true
 
-  /telejson@7.2.0:
-    resolution: {integrity: sha512-1QTEcJkJEhc8OnStBx/ILRu5J2p0GjvWsBx56bmZRqnrkdBMUe+nX92jxV+p3dB4CP6PZCdJMQJwCggkNBMzkQ==}
-    dependencies:
-      memoizerific: 1.11.3
-    dev: false
-
   /temp-dir@2.0.0:
     resolution: {integrity: sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==}
     engines: {node: '>=8'}

src/tools/sensitive-data-masker/sensitive-data-masker.service.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+import { maskSensitiveData } from './sensitive-data-masker.service';
+
+describe('sensitive-data-masker', () => {
+  describe('maskSensitiveData', () => {
+    const data = `{
+  email: 'john.doe@example.com',
+  creditCard: '1234 5678 9000 9876',
+  id: '3f8a43fd-6489-4ec7-bd55-7a1ba172d77b',
+  name: 'John',
+  surname: 'Doe',
+  phone: '+358 40 1234567',
+  url: 'truc.google.com',
+  ip4: '83.24.45.56',
+  ip6: '2001:db8:0:85a3:0:0:ac1f:8001',
+  mac: '3D:F2:C9:A6:B3:4F',
+  token: 'eyJhbGciOiJIUzI1NiJ9.ew0KICAic3ViIjogIjEyMzQ1Njc4OTAiLA0KICAibmFtZSI6ICJBbGV4IEtvemxvdiIsDQogICJpYXQiOiAxNTE2MjM5MDIyDQp9.PNKysYFTCenU5bekHCmwIxCUXoYG41H_xc3uN3ZF_b8',
+}`;
+
+    it('should maks sensitive data', () => {
+      expect(maskSensitiveData(
+        data,
+      )).toBe(`{
+  email: 'jo****************om',
+  creditCard: '12***************76',
+  id: '3f********************************7b',
+  name: 'John',
+  surname: 'Doe',
+  phone: '+35**********67',
+  url: 'tr***********om',
+  ip4: '83*******56',
+  ip6: '20*************************01',
+  mac: '3D*************4F',
+  token: 'ey*****************************************************************************************************************************************************************b8',
+}`);
+    });
+    it('should maks sensitive data (with custom regex)', () => {
+      expect(maskSensitiveData(
+        data,
+        'John\nDoe',
+      )).toBe(`{
+  email: 'jo****************om',
+  creditCard: '12***************76',
+  id: '3f********************************7b',
+  name: '****',
+  surname: '***',
+  phone: '+35**********67',
+  url: 'tr***********om',
+  ip4: '83*******56',
+  ip6: '20*************************01',
+  mac: '3D*************4F',
+  token: 'ey*****************************************************************************************************************************************************************b8',
+}`);
+    });
+  });
+});

src/tools/sensitive-data-masker/sensitive-data-masker.service.ts

@@ -0,0 +1,22 @@
+import { maskString } from 'data-guardian';
+
+const jwtRegex = /\b([a-zA-Z0-9_=]+)\.([a-zA-Z0-9_=]+)\.([a-zA-Z0-9_\-\+\/=]*)\b/g;
+const phoneRegex = /\b(?:(\+\d{1,4})[-.\s]?)?(?:[(](\d{1,3})[)][-.\s]?)?(\d{1,4})[-.\s]?(\d{1,4})[-.\s]?(\d{1,9})\b/g;
+const macRegex = /\b([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})\b/g;
+const ipv6Regex = /\b(?:(::|[0-9a-fA-F]{1,4}:{1,2})([0-9a-fA-F]{1,4}:{1,2}){0,6}([0-9a-fA-F]{1,4}|::)?)\b/g;
+const urlWithOrWithoutPrefixRegex = /\b(https?:\/\/)?(www\\.)?[-a-zA-Z0-9@:%.\_\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%\_\\+.~#?&//=]\*)\b/g;
+
+export function maskSensitiveData(value: string, customRegex?: string) {
+  return maskString(value, null as never, {
+    customRegex: new RegExp((customRegex || '').split('\n').map(line => `(${line})`).join('|'), 'gi'),
+    macRegex,
+    ipv6Regex,
+    urlWithOrWithoutPrefixRegex,
+    jwtRegex,
+    phoneRegex,
+  }, {
+    excludeMatchers: [
+      'passwordMention', 'password', 'passwordSubstring',
+    ],
+  });
+}

src/tools/sensitive-data-masker/sensitive-data-masker.vue

@@ -1,29 +1,48 @@
 <script setup lang="ts">
-import maskSensitiveData from 'mask-sensitive-data';
+import { maskSensitiveData } from './sensitive-data-masker.service';
 import { withDefaultOnError } from '@/utils/defaults';
 
-const defaultValue = `
+const defaultValue = `{
-{
   email: 'john.doe@example.com',
   creditCard: '1234 5678 9000 9876',
   id: '3f8a43fd-6489-4ec7-bd55-7a1ba172d77b',
   name: 'John',
   surname: 'Doe',
   phone: '+358 40 1234567',
+  url: 'truc.google.com',
+  ip4: '83.24.45.56',
+  ip6: '2001:db8:0:85a3:0:0:ac1f:8001',
+  mac: '3D:F2:C9:A6:B3:4F',
   token: 'eyJhbGciOiJIUzI1NiJ9.ew0KICAic3ViIjogIjEyMzQ1Njc4OTAiLA0KICAibmFtZSI6ICJBbGV4IEtvemxvdiIsDQogICJpYXQiOiAxNTE2MjM5MDIyDQp9.PNKysYFTCenU5bekHCmwIxCUXoYG41H_xc3uN3ZF_b8',
-}
+}`;
-`;
+
+const customRegex = useStorage('sensitive-data:regex', '');
+
 function transformer(value: string) {
-  return withDefaultOnError(() => maskSensitiveData(value), '');
+  return withDefaultOnError(() => maskSensitiveData(
+    value,
+    customRegex.value,
+  ), '');
 }
 </script>
 
 <template>
-  <format-transformer
+  <div style="max-width: 600px;">
-    input-label="Your log/textual data:"
+    <c-input-text
-    :input-default="defaultValue"
+      v-model:value="customRegex"
-    input-placeholder="Paste your log/textual data here..."
+      label="Your custom cleaning regex(es) (case insensitive):"
-    output-label="Cleaned version:"
+      placeholder="Your custom cleaning regex(es)"
-    :transformer="transformer"
+      raw-text
-  />
+      multiline
+      rows="4"
+    />
+
+    <format-transformer
+      input-label="Your log/textual data:"
+      :input-default="defaultValue"
+      input-placeholder="Paste your log/textual data here..."
+      output-label="Cleaned version:"
+      :transformer="transformer"
+    />
+  </div>
 </template>