Skip to content
Snippets Groups Projects
Commit bab62877 authored by Štěpán Šonovský's avatar Štěpán Šonovský
Browse files

fix: big repos not crashing csv size

parent 79ae9ceb
No related branches found
No related tags found
No related merge requests found
...@@ -11,6 +11,8 @@ ...@@ -11,6 +11,8 @@
"@gitbeaker/rest": "^40.0.3", "@gitbeaker/rest": "^40.0.3",
"@tanstack/react-query": "^5.51.1", "@tanstack/react-query": "^5.51.1",
"axios": "^1.7.2", "axios": "^1.7.2",
"axios-retry": "^4.5.0",
"es6-promise-pool": "^2.5.0",
"next": "14.2.4", "next": "14.2.4",
"react": "^18", "react": "^18",
"react-csv": "^2.2.2", "react-csv": "^2.2.2",
...@@ -989,6 +991,17 @@ ...@@ -989,6 +991,17 @@
"proxy-from-env": "^1.1.0" "proxy-from-env": "^1.1.0"
} }
}, },
"node_modules/axios-retry": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-4.5.0.tgz",
"integrity": "sha512-aR99oXhpEDGo0UuAlYcn2iGRds30k366Zfa05XWScR9QaQD4JYiP3/1Qt1u7YlefUOK+cn0CcwoL1oefavQUlQ==",
"dependencies": {
"is-retry-allowed": "^2.2.0"
},
"peerDependencies": {
"axios": "0.x || 1.x"
}
},
"node_modules/axobject-query": { "node_modules/axobject-query": {
"version": "3.1.1", "version": "3.1.1",
"resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-3.1.1.tgz", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-3.1.1.tgz",
...@@ -1627,6 +1640,14 @@ ...@@ -1627,6 +1640,14 @@
"url": "https://github.com/sponsors/ljharb" "url": "https://github.com/sponsors/ljharb"
} }
}, },
"node_modules/es6-promise-pool": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/es6-promise-pool/-/es6-promise-pool-2.5.0.tgz",
"integrity": "sha512-VHErXfzR/6r/+yyzPKeBvO0lgjfC5cbDCQWjWwMZWSb6YU39TGIl51OUmCfWCq4ylMdJSB8zkz2vIuIeIxXApA==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/escape-string-regexp": { "node_modules/escape-string-regexp": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
...@@ -2872,6 +2893,17 @@ ...@@ -2872,6 +2893,17 @@
"url": "https://github.com/sponsors/ljharb" "url": "https://github.com/sponsors/ljharb"
} }
}, },
"node_modules/is-retry-allowed": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz",
"integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/is-set": { "node_modules/is-set": {
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.3.tgz", "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.3.tgz",
......
...@@ -12,6 +12,8 @@ ...@@ -12,6 +12,8 @@
"@gitbeaker/rest": "^40.0.3", "@gitbeaker/rest": "^40.0.3",
"@tanstack/react-query": "^5.51.1", "@tanstack/react-query": "^5.51.1",
"axios": "^1.7.2", "axios": "^1.7.2",
"axios-retry": "^4.5.0",
"es6-promise-pool": "^2.5.0",
"next": "14.2.4", "next": "14.2.4",
"react": "^18", "react": "^18",
"react-csv": "^2.2.2", "react-csv": "^2.2.2",
......
import { useQuery } from "@tanstack/react-query"; import { useQuery } from "@tanstack/react-query";
import axios from "axios"; import axios from "axios";
import PromisePool from 'es6-promise-pool';
export const useCommitSize = (projectId, commitIds, token, host) => { export const useCommitSize = (projectId, commits, token, host) => {
const platform = host.includes('gitlab') ? 'gitlab' : 'github'; const platform = host.includes('gitlab') ? 'gitlab' : 'github';
const { data, isLoading } = useQuery({ const { data, isLoading } = useQuery({
queryKey: ['commitDataSize', platform, projectId, commitIds], queryKey: ['commitDataSize', platform, projectId, commits],
queryFn: async () => { queryFn: async () => {
const promises = commitIds.map(commitId => { const BATCH_SIZE = 30;
const fetchCommitData = async (commit) => {
const url = platform === 'gitlab' const url = platform === 'gitlab'
? `${host}/api/v4/projects/${projectId}/repository/commits/${commitId.id}` ? `${host}/api/v4/projects/${projectId}/repository/commits/${commit.id}`
: `https://api.github.com/repos/${projectId}/commits/${commitId.sha}`; : `https://api.github.com/repos/${projectId}/commits/${commit.sha}`;
const headers = platform === 'gitlab' const headers = platform === 'gitlab'
? { 'PRIVATE-TOKEN': token } ? { 'PRIVATE-TOKEN': token }
: { 'Authorization': `Bearer ${token}` }; : { 'Authorization': `Bearer ${token}` };
return axios.get(url, { headers }).then(response => { const response = await axios.get(url, { headers });
const { stats, commit, author } = response.data; const { stats, commit: commitData, author } = response.data;
let commitSize = 'small'; let commitSize = 'small';
const totalChanges = stats.additions + stats.deletions; const totalChanges = stats.additions + stats.deletions;
if (totalChanges > 50) commitSize = 'medium'; if (totalChanges > 50) commitSize = 'medium';
if (totalChanges > 200) commitSize = 'big'; if (totalChanges > 200) commitSize = 'big';
return { return {
project_id: projectId, project_id: projectId,
author_name: platform === 'gitlab' author_name: platform === 'gitlab'
? response.data.author_name ? response.data.author_name
: author?.login || commit.author.name, : author?.login || commitData.author.name,
created_at: platform === 'gitlab' created_at: platform === 'gitlab'
? response.data.created_at ? response.data.created_at
: commit.author.date, : commitData.author.date,
size: commitSize size: commitSize
}; };
}); };
});
const results = [];
return Promise.all(promises); const pool = new PromisePool(() => {
const commit = commits.shift();
return commit ? fetchCommitData(commit).then(result => {
results.push(result);
return result;
}) : null;
}, BATCH_SIZE);
await pool.start();
console.log('results', results);
return Promise.all(results);
} }
}); });
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment