from collections import defaultdict from itertools import groupby, chain from operator import itemgetter from django.contrib.postgres.aggregates import ArrayAgg from django.db import transaction from django.db.models import Min, F, Sum, Value, IntegerField, Count, Q from django.db.models.functions import Coalesce from rest_framework import serializers, status from api.helper.resource_map_helper import helper_recalculate_and_get_redis_resource_map from api.models import TtLocation, TtLocationSuitability, TtSetting, TtPos, TtPosModuleGroupModule, TtPathway, \ TtActivity, TtStudentSetModule, TtStudentSetActivity, TtModule, TtActivityTemplate from api.models.audit_trail import AuditTrail from api.models.audit_trail_details import AuditTrailDetails from api.models.tt_student_set import TtStudentSet from api.views.admin.base import AdminApiBase from api.validator import BaseValidator from api.utils import decrypt_aes_128_cbc, encrypt_aes_128_cbc, log_critical_error, get_exception_detail, \ get_add_new_data, get_ip, bulk_sync_to_redis, get_resource_map_redis_data from rest_framework.exceptions import ValidationError from api.translation import __ from backend.kafka import send_request from backend.redis_client import redis_client class ActivityTemplateAllocator(AdminApiBase): def validate_request(self,request): rules = { "id": "required|array|exists:api.TtActivityTemplate,id", "allocate_data": "required|array", } # if need rename field, can put here attribute = { # "email": __("attr.email"), } validator = BaseValidator(request.data,rules,attribute) error = validator.validate() if error: raise ValidationError(error) # need validate the data in allocate_data # make sure all allcate_data have student_set_ids and activity allocate_data = request.data['allocate_data'] for val in allocate_data: if "activity_id" not in val or "student_set_ids" not in val: raise ValidationError({ "error": __('validation.invalid_format'), "errors": {"allocate_data": __("validation.invalid_format")} }) activity_template_data = ( TtActivityTemplate.objects.filter(id__in=request.data['id']) .annotate( student_set_ids=ArrayAgg( "module__ttstudentsetmodule__student_set_id", distinct=True ), ) .prefetch_related('ttactivity_set') ) # activity id must in available_activity_ids available_activity_ids = [] # used to get name incase want return name to FE activity_data = {} # used to compare FE input student_set_ids value must exists in this activity_available_student_set activity_available_student_set = {} for template_info in activity_template_data: for activity in template_info.ttactivity_set.all(): available_activity_ids.append(activity.id) activity_data[activity.id] = activity activity_available_student_set[activity.id] = template_info.student_set_ids or [] for val in allocate_data: # check activity id is available if val['activity_id'] not in available_activity_ids: raise ValidationError({ "error": __('validation.activity_template_allocator_custom_msg.activity_id_not_found',value=val['activity_id']), "errors": {"allocate_data": __('validation.activity_template_allocator_custom_msg.activity_id_not_found',value=val['activity_id'])} }) student_set_ids = val.get('student_set_ids', []) available_student_sets = activity_available_student_set.get(val['activity_id'], []) # check student set is available for that activity not_found_ids = [ss_id for ss_id in student_set_ids if ss_id not in available_student_sets] if not_found_ids: raise ValidationError({ "error": __('validation.activity_template_allocator_custom_msg.student_set_ids_not_found',value=not_found_ids), "errors": {"allocate_data": __('validation.activity_template_allocator_custom_msg.student_set_ids_not_found',value=not_found_ids)} }) def post(self, request): try: self.validate_request(request) setting_params = { "slot_per_week", } settings = TtSetting.get_multiple_setting(setting_params) slot_per_week = int(settings["slot_per_week"]) old_new_data = { "old_data": [], "new_data": [], } action="activity_template_allocate" # this is for audit log only allocate_data = request.data["allocate_data"] allocate_data_set = {item["activity_id"]: item["student_set_ids"] for item in allocate_data} redis_activity_table = "activity" redis_student_set_table = "student_set" redis_data = { "update": {} } # get all student set activity relation activity_ids = [item['activity_id'] for item in allocate_data] # check activity_ids got any variant child or not, will update it same as variant parent variant_activities = TtActivity.objects.filter(variant_parent__in=activity_ids, is_variant=1) if variant_activities: for variant_activity in variant_activities: # add to activity_ids activity_ids.append(variant_activity.id) # append variant child id to allocate_data, so no need change the code for handle variant allocate_data.append({ "activity_id": variant_activity.id, "student_set_ids": allocate_data_set[variant_activity.variant_parent_id], }) activities = TtActivity.objects.filter(id__in=activity_ids).prefetch_related("student_set") # at the end to check need update db is_jta_activity = [] student_set_data = defaultdict(list) activity_data = {} activity_with_student_set = defaultdict(list) affected_activity_ids = [] new_relation_data = [] # reformat data for audit trail purpose for val in activities: activity_data[val.id] = val current_student_sets = val.student_set.all() activity_with_student_set[val.id] = [s.id for s in current_student_sets] for s in current_student_sets: student_set_data[s.id] = s affected_student_set_ids = set() # compare which activity got adjust the student set for val in allocate_data: old_student_set_ids = activity_with_student_set[val['activity_id']] new_student_set_ids = val['student_set_ids'] current_activity_data = activity_data[val['activity_id']] if current_activity_data.is_jta: is_jta_activity.append(current_activity_data) # if current system data got different with latest FE pass in data, only update relation for that activity if set(old_student_set_ids) != set(new_student_set_ids): # got different only need update for resources map affected_student_set_ids.update(set(old_student_set_ids)) affected_student_set_ids.update(set(new_student_set_ids)) old_new_data["old_data"].append({ "activity_name": current_activity_data.name if current_activity_data else None, "student_set_ids": old_student_set_ids, }) old_new_data["new_data"].append({ "activity_name": current_activity_data.name if current_activity_data else None, "student_set_ids": new_student_set_ids, }) # record which id need to delete relation, later will used 1 query delete all related relation affected_activity_ids.append(val['activity_id']) # in redis this column call student_sets redis_data["update"].setdefault(redis_activity_table, []).append({"id":val['activity_id'],"student_sets": new_student_set_ids,}) # record what new relation need to be add, so will use bulk_create function to create tgt, so can reduce the query used for new_student_set_id in new_student_set_ids: new_relation_data.append({ "activity_id": val['activity_id'], "student_set_ids": new_student_set_id, }) # remove that activity all old relation if affected_activity_ids: TtStudentSetActivity.objects.filter(activity_id__in=affected_activity_ids).delete() # print(affected_activity_ids) # add new relation if new_relation_data: # Convert to model instances idata = [ TtStudentSetActivity( activity_id=item['activity_id'], student_set_id=item['student_set_ids'] ) for item in new_relation_data ] TtStudentSetActivity.objects.bulk_create(idata, batch_size=500) # if not empty, need update these activity jta child record if is_jta_activity: jta_child_ids = {activity.jta_parent_id for activity in is_jta_activity if activity.jta_parent_id} # get child activity student_set with all parent and parent student set jta_activities = ( TtActivity.objects.filter( Q(jta_parent_id__in=jta_child_ids) | Q(id__in=jta_child_ids) ).prefetch_related("student_set") ) jta_activities_set = {act.id: act for act in jta_activities} # group jta activity to child, so later no need do looping to find student set id activities_by_child = defaultdict(list) for act in jta_activities: if act.jta_parent_id: activities_by_child[act.jta_parent_id].append(act) for jta_child_id in jta_child_ids: child_activity = jta_activities_set.get(jta_child_id) # safety check, if no child_activity atleast wont cause critical error if not child_activity: continue # child student set always follow parent, so no need get child self student set student_set_ids = set() related_activities = activities_by_child.get(jta_child_id, []) for related_activity in related_activities: # group all student_set_ids that need update to jta child activity student_set_ids.update(s.id for s in related_activity.student_set.all()) student_set_ids_list = list(student_set_ids) child_activity.student_set.set(student_set_ids) redis_data["update"].setdefault(redis_activity_table, []).append({"id":child_activity.id,"student_sets": student_set_ids_list,}) # if the child activity is variant also, need update that variant activity if child_activity.is_variant: child_variant_activities = TtActivity.objects.filter(variant_parent_id=child_activity.id) for child_variant_activity in child_variant_activities: child_variant_activity.student_set.set(student_set_ids) redis_data["update"].setdefault(redis_activity_table, []).append({"id": child_variant_activity.id, "student_sets": student_set_ids_list,}) student_set_with_affected_week = {} kafka_allocator = [] # update resource map and redis data if affected_student_set_ids: student_sets = TtStudentSet.objects.filter(id__in=affected_student_set_ids).prefetch_related("resource_map", "activity") for student_set in student_sets: week_ids = [ss_week.id for ss_week in student_set.resource_map.all()] if student_set.id not in student_set_with_affected_week: student_set_with_affected_week[student_set.id] = set() student_set_with_affected_week[student_set.id].update(week_ids) # kafka if student_set.activity: kafka_data = { "student_set": student_set.id, "activity": [act.id for act in student_set.activity.all()] # full activity ids } kafka_allocator.append(kafka_data) student_set_with_affected_week = {k: v for k, v in student_set_with_affected_week.items() if v} # since got update student set, this helper already cater if got update student set, no need pass in student details also will update student resource map redis_data = helper_recalculate_and_get_redis_resource_map( redis_data=redis_data, student_set_with_affected_week=student_set_with_affected_week, slot_per_week=slot_per_week, ) if redis_data['update']: bulk_sync_to_redis(redis_client, redis_data) method = "activity_template_allocator" kafka_topic = self.kafka_config['MICROSERVICES_TT_TOPIC'] if kafka_topic and kafka_allocator: kafka_request_data = { "session_id": request.user.name, "allocator": kafka_allocator } send_request(kafka_topic, kafka_request_data, None, method) # insert audit trail and details # can insert the parent first, then only insert details audit_trail = AuditTrail.objects.create( user_id=request.user.id, type=self.audit_type, ip_address=get_ip(request) ) remark_param = { # "code":student_set.code, } # call to a function for insert details, so in future if want change to used worker, can modify in function only AuditTrailDetails.custom_insert( audit_trail=audit_trail, action=action, remark_param=remark_param, new_data=old_new_data['new_data'], old_data=old_new_data['old_data'] ) response = { } return self.api_response(data=response) except ValidationError as e: first_message = e.detail['error'] errors = e.detail['errors'] return self.api_response(error=first_message,errors=errors,code=status.HTTP_400_BAD_REQUEST) except Exception as e: e_details = get_exception_detail(e) log_critical_error(user_id=None,descr=e_details['descr'],url=e_details['url'],trace=e_details['trace']) return self.api_response(error=__("message.internal_server_error"),code=status.HTTP_500_INTERNAL_SERVER_ERROR)