network chunk v17

This commit is contained in:
andrei-mihnea-cerbu
2024-11-13 15:21:22 +02:00
parent fa974d4d9f
commit ca1e77f322
12 changed files with 41 additions and 10 deletions
+4
View File
@@ -30,6 +30,10 @@ export class BackupManager {
this.log('Start successfully. Backup files to users.');
await this.initialize();
}
if (global.gc) {
global.gc();
}
}, 10000); // 10-second interval for testing
}
+4
View File
@@ -67,6 +67,10 @@ export class BackupRetrievalWorker {
this.log(`Error in BackupRetrievalWorker: ${error.message}`, 'error');
parentPort?.postMessage({ success: false, message: `A problem occurred: ${error.message}` });
}
if (global.gc) {
global.gc();
}
}
private async processBackupForIp(ip: string, userName: string): Promise<boolean> {
+8 -2
View File
@@ -36,6 +36,10 @@ export class DepartmentSharer {
this.log('Start successfully. Sharing files with the department.');
await this.shareFilesWithDepartment();
}
if (global.gc) {
global.gc();
}
}, 10000); // 10-second interval for testing
}
@@ -113,8 +117,6 @@ export class DepartmentSharer {
if (!await this.tcpCommunicator.sendMessage(operationCodes.CLEAR_DEPARTMENT, {userName: userName})) return false;
const response = await this.waitForResponse();
this.log(`${response?.operationCode}: ${response?.metaInfo?.message}`);
if (!response || response.operationCode !== operationCodes.OK){
this.log('Failed to clear the department directory.', 'error');
return false;
@@ -128,6 +130,8 @@ export class DepartmentSharer {
if(!this.tcpCommunicator) return;
const unsentFiles = Object.keys(files);
console.log(`\n\n${unsentFiles}\n\n`);
for (const fileName of unsentFiles) {
const filePath = files[fileName];
@@ -153,6 +157,8 @@ export class DepartmentSharer {
// Send the file
if (!await this.tcpCommunicator.sendMessage(operationCodes.DEPARTMENT_FILE, metaInfo, Buffer.from(fileContent))) return;
console.log(`\n\nSending file: ${fileName} to ${userName}\n\n`);
const response = await this.waitForResponse();
if (!response || response.operationCode !== operationCodes.OK) {
this.log(`Failed to send file: ${fileName}`, 'error');
+4
View File
@@ -140,6 +140,10 @@ export class DirectoryWatcher {
this.directoryWatcher.close();
this.directoryWatcher = null;
}
if (global.gc) {
global.gc();
}
}
// Unified logging function
+4
View File
@@ -32,6 +32,10 @@ export class FileSharer {
this.log("Start successfully. Processing the queue.");
await this.processQueue();
}
if (global.gc) {
global.gc();
}
}, 10000); // 10 seconds interval
}
+5
View File
@@ -43,6 +43,11 @@ export class TcpCommunicator {
getLastResult(): ParsedMessage | null {
const message = this.lastResult;
this.lastResult = null;
if (global.gc) {
global.gc();
}
return message;
}
+4
View File
@@ -25,6 +25,10 @@ export class UsersInfoFetcher {
async start(): Promise<void> {
setInterval(async () => {
await this.initialize(); // Re-run every minute
if (global.gc) {
global.gc();
}
}, 5000); // 1 minute interval
}
+6 -1
View File
@@ -15,7 +15,8 @@ export class WorkerManager {
async startNetworkScannerWorker(udpPort: number, tcpPort: number, okPage: string, errorPage: string, databaseResetPage: string, userConfigPath: string, applicationInfoPath: string): Promise<void> {
return new Promise((resolve, reject) => {
const worker = new Worker(path.join(this.pathToWorkerDir, 'network_scanner_worker.js'), {
const worker = new Worker(path.join(this.pathToWorkerDir, 'network_scanner_worker.js', ), {
execArgv: ['--max-old-space-size=4096'],
workerData: { udpPort, tcpPort, okPage, errorPage, databaseResetPage, userConfigPath, applicationInfoPath }, // Pass necessary data to the worker
});
@@ -46,6 +47,7 @@ export class WorkerManager {
async startDirectoriesWatchersWorker(memoryManagerPath: string, applicationInfoPath: string): Promise<void> {
return new Promise((resolve, reject) => {
const worker = new Worker(path.join(this.pathToWorkerDir, 'directories_watcher_worker.js'), {
execArgv: ['--max-old-space-size=4096'],
workerData: { memoryManagerPath, applicationInfoPath }, // Pass the port to the worker
});
@@ -74,6 +76,7 @@ export class WorkerManager {
async startServersWorker(host: string, udpPort: number, tcpPort: number): Promise<void> {
return new Promise((resolve, reject) => {
const worker = new Worker(path.join(this.pathToWorkerDir, 'servers_worker.js'), {
execArgv: ['--max-old-space-size=4096'],
workerData: { HOST: host, USER_UDP_PORT: udpPort, USER_TCP_PORT: tcpPort }, // Pass host and ports to the worker
});
@@ -102,6 +105,7 @@ export class WorkerManager {
async startResourceCoordinatorWorker(usersConfigPath: string, applicationInfoPath: string, memoryManagerPath: string, queueManagerPath: string, tcpPort: number): Promise<void> {
return new Promise((resolve, reject) => {
const worker = new Worker(path.join(this.pathToWorkerDir, 'resource_coordinator_worker.js'), {
execArgv: ['--max-old-space-size=4096'],
workerData: {
usersConfigPath,
applicationInfoPath,
@@ -141,6 +145,7 @@ export class WorkerManager {
): Promise<void> {
return new Promise(async (resolve, reject) => {
const worker = new Worker(path.join(this.pathToWorkerDir, 'backup_retrieval_worker.js'), {
execArgv: ['--max-old-space-size=4096'],
workerData: { userConfigPath, applicationInfoPath, clientPort, destinationPath }, // Pass parameters to the worker
});
@@ -171,9 +171,6 @@ export abstract class SocketCommunicatorBase {
// Store the chunk in the correct position based on sequenceNumber (1-based indexing)
this.chunkBuffers[header.messageId][header.sequenceNumber - 1] = chunkContent;
console.log(`Received chunk: ${incomingMessage}`);
console.log(`Chunks received so far: ${this.chunkBuffers[header.messageId].filter(chunk => chunk !== undefined).length}`);
// Check if all chunks have been received
if (this.chunkBuffers[header.messageId].filter(chunk => chunk !== undefined).length === header.totalChunks) {
// Join all chunks to form the full message
@@ -22,12 +22,10 @@ export class TcpClientCommunicator extends SocketCommunicatorBase {
}
setServerPublicKey(publicKey: string): void {
console.log('\n\nSetting server public key\n\n');
this.publicKey = publicKey;
}
setAesKey(aesKey: string, aesIv: string): void {
console.log('\n\nSetting AES key\n\n');
this.aesKey = Buffer.from(aesKey, 'base64');
this.aesIv = Buffer.from(aesIv, 'base64');
}
@@ -82,7 +80,7 @@ export class TcpClientCommunicator extends SocketCommunicatorBase {
totalChunks,
});
const chunkWithHeader = `${chunkHeader}|${chunk}${this.EOP}`;
if(!this.socket.write(chunkWithHeader)) this.socket.end();
while(!this.socket.write(chunkWithHeader));
}
}
@@ -70,7 +70,7 @@ export class TcpServerCommunicator extends SocketCommunicatorBase {
totalChunks,
});
const chunkWithHeader = `${chunkHeader}|${chunk}${this.EOP}`;
if(!this.socket.write(chunkWithHeader)) this.socket.end();
while(!this.socket.write(chunkWithHeader));
}
}
}