Merge branch '512-optimize-technical-exercise-creation' into 'main' (47644a43) · Commits · INJECT / backend

running_exercise/lib/opensearch_client.py

+71 −49

Original line number	Diff line number	Diff line
		from opensearchpy import OpenSearch
		from opensearchpy import OpenSearch, helpers
		from django.conf import settings
		from exercise.models import OpenSearchAccess

		@@ -15,6 +15,7 @@ def get_opensearch_client():
		http_auth=auth,
		use_ssl=True,
		verify_certs=False,
		timeout=300, # headroom for bulk operations
		)


		@@ -33,66 +34,87 @@ def create_opensearch_exercise(teams, exercise_id):
		"""
		client = get_opensearch_client()

		opensearch_access_list = []

		# TODO: optimise for large number of teams (query fails for timeout) => bulk API?
		for team in teams:
		team_id = team.id

		username = f"exercise-{exercise_id}--team-{team_id}"
		index_name = f"{username}--index"
		role_name = f"{username}--role"
		# TODO: ensure that the password will be strong enough to pass the OpenSearch verification
		password = PassphraseGenerator.generate_passphrase()
		team_data_list = [
		{
		"team": team,
		"username": f"exercise-{exercise_id}--team-{team.id}",
		"index_name": f"exercise-{exercise_id}--team-{team.id}--index",
		"role_name": f"exercise-{exercise_id}--team-{team.id}--role",
		"password": PassphraseGenerator.generate_passphrase(),
		}
		for team in teams
		]

		opensearch_access_list.append(
		OpenSearchAccess(
		team=team,
		index_name=index_name,
		username=username,
		password=password,
		try:
		indices_body = [
		{
		"_op_type": "create",
		"_index": team_data["index_name"],
		}
		for team_data in team_data_list
		]
		_, failed_operations = helpers.bulk(
		client, indices_body, raise_on_error=False
		)
		if failed_operations:
		raise Exception(
		f"Failed to create some indices: {failed_operations}"
		)

		try:
		client.indices.create(index=index_name)

		# https://docs.opensearch.org/docs/latest/security/access-control/users-roles/#defining-users
		# https://docs.opensearch.org/docs/latest/security/access-control/api/#create-user
		client.security.create_user(
		username=username, body={"password": password}
		)
		users_body = [
		{
		"op": "add",
		"path": f"/{team_data['username']}",
		"value": {"password": team_data["password"]},
		}
		for team_data in team_data_list
		]
		client.security.patch_users(body=users_body)

		# https://docs.opensearch.org/docs/latest/security/access-control/users-roles/#defining-roles
		# https://docs.opensearch.org/docs/latest/security/access-control/api/#create-role
		# https://docs.opensearch.org/docs/latest/security/access-control/permissions/
		client.security.create_role(
		role=role_name,
		body={
		roles_body = [
		{
		"op": "add",
		"path": f"/{team_data['role_name']}",
		"value": {
		"cluster_permissions": [
		"cluster:monitor/main", # high-level monitoring of cluster state; necessary for Logstash
		],
		"index_permissions": [
		{
		"index_patterns": [f"{index_name}"],
		"index_patterns": [f"{team_data['index_name']}"],
		"allowed_actions": [
		"write",
		],
		}
		],
		},
		)
		}
		for team_data in team_data_list
		]
		client.security.patch_roles(body=roles_body)

		# https://docs.opensearch.org/docs/latest/security/access-control/users-roles/#mapping-users-to-roles
		# https://docs.opensearch.org/docs/latest/security/access-control/api/#create-role-mapping
		client.security.create_role_mapping(
		role=role_name,
		body={
		"users": [username],
		role_mappings_body = [
		{
		"op": "add",
		"path": f"/{team_data['role_name']}",
		"value": {
		"users": [team_data["username"]],
		},
		)
		}
		for team_data in team_data_list
		]
		client.security.patch_role_mappings(body=role_mappings_body)
		except Exception as e:
		# TODO: cleanup and re-raise (exercise in IXP already exists)
		raise e

		OpenSearchAccess.objects.bulk_create(opensearch_access_list)
		opensearch_access_body = [
		OpenSearchAccess(
		team=team_data["team"],
		index_name=team_data["index_name"],
		username=team_data["username"],
		password=team_data["password"],
		)
		for team_data in team_data_list
		]
		OpenSearchAccess.objects.bulk_create(opensearch_access_body)