public-repos/it-tools
5
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: enhance cleaned PII and custom regexes

Browse Source
This commit is contained in:
ShareVB 2024-07-07 22:59:31 +02:00
parent e6e51cf1c4
commit 933d824083
5 changed files with 119 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -55,6 +55,7 @@
"cron-validator": "^1.3.1",
"cronstrue": "^2.26.0",
"crypto-js": "^4.1.1",
"data-guardian": "^1.1.3",
"date-fns": "^2.29.3",
"dompurify": "^3.0.6",
"emojilib": "^3.0.10",
@ -70,7 +71,6 @@
"libphonenumber-js": "^1.10.28",
"lodash": "^4.17.21",
"marked": "^10.0.0",
"mask-sensitive-data": "^0.11.5",
"mathjs": "^11.9.1",
"mime-types": "^2.1.35",
"monaco-editor": "^0.43.0",

33
pnpm-lock.yaml generated
View File

@ -65,6 +65,9 @@ dependencies:
crypto-js:
specifier: ^4.1.1
version: 4.1.1
data-guardian:
specifier: ^1.1.3
version: 1.1.3
date-fns:
specifier: ^2.29.3
version: 2.29.3
@ -110,9 +113,6 @@ dependencies:
marked:
specifier: ^10.0.0
version: 10.0.0
mask-sensitive-data:
specifier: ^0.11.5
version: 0.11.5
mathjs:
specifier: ^11.9.1
version: 11.9.1
@ -4722,6 +4722,11 @@ packages:
resolution: {integrity: sha512-4FbVrHDwfOASx7uQVxeiCTo7ggSdYZbqs8lH+WU6ViypPlDbe9y6IP5VVUDQBv9DcnyaiPT5XT0UWHgJ64zLeQ==}
dev: false
/data-guardian@1.1.3:
resolution: {integrity: sha512-8M9KHTopFp3MXo7NelcmF+8L77S5jeDy3Uv4RWuYIQSaBZTawvY6W3ad/eLe8db2FcLPoC44DHJb4zWNcpoueQ==}
engines: {node: '>=12'}
dev: false
/data-uri-to-buffer@4.0.1:
resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
engines: {node: '>= 12'}
@ -6802,10 +6807,6 @@ packages:
resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==}
dev: false
/map-or-similar@1.5.0:
resolution: {integrity: sha512-0aF7ZmVon1igznGI4VS30yugpduQW3y3GkcgGJOp7d8x8QrizhigUxjI/m2UojsXXto+jLAH3KSz+xOJTiORjg==}
dev: false
/map-stream@0.1.0:
resolution: {integrity: sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==}
dev: true
@ -6826,12 +6827,6 @@ packages:
hasBin: true
dev: false
/mask-sensitive-data@0.11.5:
resolution: {integrity: sha512-NBl/OC0kkKQaF5uo7dNnnNihidZM0TBCpxnKAHzTTmBnFPaWX9sZ2DHXOpnWRdrltgeTryTnrQ8pOutKIhkMgw==}
dependencies:
telejson: 7.2.0
dev: false
/mathjs@11.9.1:
resolution: {integrity: sha512-VdB9ELZ6Dwda13j5d1eCBETmPO8m9qIJETUdfZmTA9cPnXUiIk7UuoAmvxFqtfAe32XYuRugAec2Ndv0/RfRhg==}
engines: {node: '>= 14'}
@ -6874,12 +6869,6 @@ packages:
/mdurl@1.0.1:
resolution: {integrity: sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==}
/memoizerific@1.11.3:
resolution: {integrity: sha512-/EuHYwAPdLtXwAwSZkh/Gutery6pD2KYd44oQLhAvQp/50mpyduZh8Q7PYHXTCJ+wuXxt7oij2LXyIJOOYFPog==}
dependencies:
map-or-similar: 1.5.0
dev: false
/merge-stream@2.0.0:
resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==}
dev: true
@ -8392,12 +8381,6 @@ packages:
tslib: 2.5.0
dev: true
/telejson@7.2.0:
resolution: {integrity: sha512-1QTEcJkJEhc8OnStBx/ILRu5J2p0GjvWsBx56bmZRqnrkdBMUe+nX92jxV+p3dB4CP6PZCdJMQJwCggkNBMzkQ==}
dependencies:
memoizerific: 1.11.3
dev: false
/temp-dir@2.0.0:
resolution: {integrity: sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==}
engines: {node: '>=8'}

56
src/tools/sensitive-data-masker/sensitive-data-masker.service.test.ts Normal file
View File

@ -0,0 +1,56 @@
import { describe, expect, it } from 'vitest';
import { maskSensitiveData } from './sensitive-data-masker.service';
describe('sensitive-data-masker', () => {
describe('maskSensitiveData', () => {
const data = `{
email: 'john.doe@example.com',
creditCard: '1234 5678 9000 9876',
id: '3f8a43fd-6489-4ec7-bd55-7a1ba172d77b',
name: 'John',
surname: 'Doe',
phone: '+358 40 1234567',
url: 'truc.google.com',
ip4: '83.24.45.56',
ip6: '2001:db8:0:85a3:0:0:ac1f:8001',
mac: '3D:F2:C9:A6:B3:4F',
token: 'eyJhbGciOiJIUzI1NiJ9.ew0KICAic3ViIjogIjEyMzQ1Njc4OTAiLA0KICAibmFtZSI6ICJBbGV4IEtvemxvdiIsDQogICJpYXQiOiAxNTE2MjM5MDIyDQp9.PNKysYFTCenU5bekHCmwIxCUXoYG41H_xc3uN3ZF_b8',
}`;
it('should maks sensitive data', () => {
expect(maskSensitiveData(
data,
)).toBe(`{
email: 'jo****************om',
creditCard: '12***************76',
id: '3f********************************7b',
name: 'John',
surname: 'Doe',
phone: '+35**********67',
url: 'tr***********om',
ip4: '83*******56',
ip6: '20*************************01',
mac: '3D*************4F',
token: 'ey*****************************************************************************************************************************************************************b8',
}`);
});
it('should maks sensitive data (with custom regex)', () => {
expect(maskSensitiveData(
data,
'John\nDoe',
)).toBe(`{
email: 'jo****************om',
creditCard: '12***************76',
id: '3f********************************7b',
name: '****',
surname: '***',
phone: '+35**********67',
url: 'tr***********om',
ip4: '83*******56',
ip6: '20*************************01',
mac: '3D*************4F',
token: 'ey*****************************************************************************************************************************************************************b8',
}`);
});
});
});

22
src/tools/sensitive-data-masker/sensitive-data-masker.service.ts Normal file
View File

@ -0,0 +1,22 @@
import { maskString } from 'data-guardian';
const jwtRegex = /\b([a-zA-Z0-9_=]+)\.([a-zA-Z0-9_=]+)\.([a-zA-Z0-9_\-\+\/=]*)\b/g;
const phoneRegex = /\b(?:(\+\d{1,4})[-.\s]?)?(?:[(](\d{1,3})[)][-.\s]?)?(\d{1,4})[-.\s]?(\d{1,4})[-.\s]?(\d{1,9})\b/g;
const macRegex = /\b([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})\b/g;
const ipv6Regex = /\b(?:(::|[0-9a-fA-F]{1,4}:{1,2})([0-9a-fA-F]{1,4}:{1,2}){0,6}([0-9a-fA-F]{1,4}|::)?)\b/g;
const urlWithOrWithoutPrefixRegex = /\b(https?:\/\/)?(www\\.)?[-a-zA-Z0-9@:%.\_\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%\_\\+.~#?&//=]\*)\b/g;
export function maskSensitiveData(value: string, customRegex?: string) {
return maskString(value, null as never, {
customRegex: new RegExp((customRegex || '').split('\n').map(line => `(${line})`).join('|'), 'gi'),
macRegex,
ipv6Regex,
urlWithOrWithoutPrefixRegex,
jwtRegex,
phoneRegex,
}, {
excludeMatchers: [
'passwordMention', 'password', 'passwordSubstring',
],
});
}

31
src/tools/sensitive-data-masker/sensitive-data-masker.vue
View File

@ -1,24 +1,42 @@
<script setup lang="ts">
import maskSensitiveData from 'mask-sensitive-data';
import { maskSensitiveData } from './sensitive-data-masker.service';
import { withDefaultOnError } from '@/utils/defaults';
const defaultValue = `
{
const defaultValue = `{
email: 'john.doe@example.com',
creditCard: '1234 5678 9000 9876',
id: '3f8a43fd-6489-4ec7-bd55-7a1ba172d77b',
name: 'John',
surname: 'Doe',
phone: '+358 40 1234567',
url: 'truc.google.com',
ip4: '83.24.45.56',
ip6: '2001:db8:0:85a3:0:0:ac1f:8001',
mac: '3D:F2:C9:A6:B3:4F',
token: 'eyJhbGciOiJIUzI1NiJ9.ew0KICAic3ViIjogIjEyMzQ1Njc4OTAiLA0KICAibmFtZSI6ICJBbGV4IEtvemxvdiIsDQogICJpYXQiOiAxNTE2MjM5MDIyDQp9.PNKysYFTCenU5bekHCmwIxCUXoYG41H_xc3uN3ZF_b8',
}
`;
}`;
const customRegex = useStorage('sensitive-data:regex', '');
function transformer(value: string) {
return withDefaultOnError(() => maskSensitiveData(value), '');
return withDefaultOnError(() => maskSensitiveData(
value,
customRegex.value,
), '');
}
</script>
<template>
<div style="max-width: 600px;">
<c-input-text
v-model:value="customRegex"
label="Your custom cleaning regex(es) (case insensitive):"
placeholder="Your custom cleaning regex(es)"
raw-text
multiline
rows="4"
/>
<format-transformer
input-label="Your log/textual data:"
:input-default="defaultValue"
@ -26,4 +44,5 @@ function transformer(value: string) {
output-label="Cleaned version:"
:transformer="transformer"
/>
</div>
</template>