from collections import defaultdict from itertools import groupby, chain from operator import itemgetter from django.contrib.postgres.aggregates import ArrayAgg from django.db import transaction from django.db.models import Min, F, Sum, Value, IntegerField, Count, Q from django.db.models.functions import Coalesce from django.forms.models import model_to_dict from rest_framework import serializers, status from api.helper.resource_map_helper import helper_recalculate_and_get_redis_resource_map from api.models import TtLocation, TtLocationSuitability, TtSetting, TtPos, TtPosModuleGroupModule, TtPathway, \ TtActivity, TtStudentSetModule, TtStudentSetActivity, TtModule, TtActivityTemplate, TtWeek, TtStudentSetResourceMap, \ TtAcademicTerm, TtStudent, TtStudentSetStudent from api.models.audit_trail import AuditTrail from api.models.audit_trail_details import AuditTrailDetails from api.models.tt_student_set import TtStudentSet from api.views.admin.base import AdminApiBase from api.validator import BaseValidator from api.utils import decrypt_aes_128_cbc, encrypt_aes_128_cbc, log_critical_error, get_exception_detail, \ get_add_new_data, get_ip, bulk_sync_to_redis, get_missing_code_from_db, get_auto_generate_name, get_missing_name, \ get_slot_default_pattern, get_resource_map_redis_data, get_auto_generate_code from rest_framework.exceptions import ValidationError from api.translation import __ from backend.kafka import send_request from backend.redis_client import redis_client class Allocation(AdminApiBase): def validate_request(self,request): rules = { "type": "required", # pathway/module.programme "allocation_method": "required", # spread/clump } # if need rename field, can put here attribute = { # "email": __("attr.email"), } validator = BaseValidator(request.data,rules,attribute) error = validator.validate() if error: raise ValidationError(error) check_table = "" if request.data['type'] == "pathway": check_table = "TtPathway" rules = { "id": "required|array|exists:api."+check_table+",id",# depends on type, see want check from which table } # if need rename field, can put here attribute = { # "email": __("attr.email"), } validator = BaseValidator(request.data,rules,attribute) error = validator.validate() if error: raise ValidationError(error) def post(self, request): # Validate input try: setting_params = { "slot_per_week", } settings = TtSetting.get_multiple_setting(setting_params) slot_per_week = int(settings["slot_per_week"]) self.validate_request(request) old_new_data = { "old_data": None, "new_data": None, } slot_default_pattern = get_slot_default_pattern(slot_per_week) action="allocation" # this is for audit log only type = request.data['type'] allocation_method = request.data['allocation_method'] ids = request.data['id'] batch_size = 500 return_data = [] student_set_generated_count = 0 failed = [] kafka_student_set = [] kafka_student_allocate_data = {} redis_student_set_table = "student_set" redis_activity_table = "activity" redis_student_table = "student" allocated_student_count = 0 recalculate_student_ids = set() recalculate_week_ids = set() # if this set not empty, will have 1 query get related activity student ids to update redis data affected_activity_ids = set() redis_data = { "insert": {}, "update": {}, } # loop based on activity planned size # no more activity or finish loop of pathway planned size will stop looping if type == "pathway": related_pathway_data = get_related_pathway(ids) all_pathway_remaining_size = calculate_pathway_remaining_size(related_pathway_data) """ REMARK Different from activity_generate to handle missing code and name, cause pathway pos_id is not arrange, so will used array to set every pos_id until which name/code """ pos_ids = list(related_pathway_data.values_list("pos_id", flat=True).distinct()) all_pos_name = list(related_pathway_data.values_list("pos_name",flat=True).distinct()) # map the pos_id with the pos_name, so when find missing name can used pos_id_name = list(related_pathway_data.values("pos_id","pos_name")) academic_term_ids = list(related_pathway_data.values_list("academic_term_id", flat=True).distinct()) # used for create student set resource map weeks_data = TtWeek.objects.filter(ttacademicterm__id__in=academic_term_ids).order_by("week").distinct() # recalculate_week_ids will used for calculate resource map for student recalculate_week_ids.update({w.id for w in weeks_data}) pos_id_name_map = {} for item in pos_id_name: pos_id_name_map[item["pos_id"]] = item["pos_name"] # used for query from tt_student_set to get all name start with all the prefix all_student_set_name_prefix = [] for pos_name in all_pos_name: all_student_set_name_prefix.append(pos_name+self.auto_generate_name_seperator) # somethings like "OR" query to filter out multiple prefix prefix_query = Q() for p in all_student_set_name_prefix: prefix_query |= Q(name__startswith=p) current_student_set = ( TtStudentSet.objects.filter(pos_id__in=pos_ids) .filter(prefix_query) .values("pos_id","name") ) existing_student_set_name = {} for val in current_student_set: existing_student_set_name.setdefault(val["pos_id"], []).append(val["name"]) last_student_set_name = {} missing_student_set_name = {} for key,val in existing_student_set_name.items(): if val: last_student_set_name.setdefault(key,None) last_student_set_name[key] = max((n for n in val), key=lambda x: int(x.split(self.auto_generate_name_seperator)[-1])) missing_student_set_name.setdefault(key,[]).extend(get_missing_name(val, pos_id_name_map[key]+self.auto_generate_name_seperator, 2)) missing_code = get_missing_code_from_db(self.default_code_prefix, "tt_student_set", 10) last_student_set = TtStudentSet.objects.filter(code__startswith=self.default_code_prefix).order_by('-code').first() last_code = last_student_set.code if last_student_set else None # filter all module_ids in all pathway record, and only search related module_ids in activity table to reduce the record return for module_have_activity all_pathway_module_ids = list( set(chain.from_iterable(p["module_ids"] for p in related_pathway_data if p["module_ids"]))) module_have_template = set(TtActivityTemplate.objects.filter(module_id__in=all_pathway_module_ids,module_id__isnull=False).values_list("module_id", flat=True).distinct()) # get all related module with activity template and activity count for display err msg related_activity_template_data = list( TtActivityTemplate.objects .filter(module_id__in=all_pathway_module_ids) .select_related("module") .annotate( activity_count=Count("ttactivity", distinct=True) # count related activities ) .values( "id", "code", "name", "module_id", "module__code", "module__name", "activity_count" ) ) related_module_data = { item["module_id"]: { "module_id": item["module_id"], "module_code": item["module__code"], "module_name": item["module__name"], } for item in related_activity_template_data } related_activity_template_data = { item["id"]: { "activity_template_id": item["id"], "activity_template_code": item["code"], "activity_template_name": item["name"], "module_id": item["module_id"], "module_code": item["module__code"], "module_name": item["module__name"], "activity_count": item["activity_count"], } for item in related_activity_template_data } academic_term_key_by_id = TtAcademicTerm.objects.in_bulk(academic_term_ids) pathway_names = {} for pathway_data in related_pathway_data: student_set_student_relation_idata = [] pos_name = pathway_data["pos_name"] pathway_names[pos_name] = None pathway_student_set_ids = pathway_data["student_set_ids"] has_failed = False student_ids = [] # every pathway also get 1 time student, so if got student allocate to student set in previous pathway, wont allocate again # pathway_students = TtStudent.objects.filter(pathway__id=pathway_data["id"]).prefetch_related("student_set") # after assign to student set, will keep remove last one, until it is empty or all student set is full # for pathway_student in pathway_students: # student_set_in_pos = False # for student_set_student in pathway_student.student_set.all(): # if student_set_student.pos_id == pathway_data["pos_id"]: # student_set_in_pos = True # break # if not student_set_in_pos: # student_ids.append(pathway_student.id) pathway_students = list( TtStudent.objects.filter( pathway__id=pathway_data["id"] ) .exclude(student_set__id__in=pathway_student_set_ids) .prefetch_related("student_set") # 2026-09-11 based on client structure, student can allocate to multiple student set in 1 same term, so no need exclude # .exclude( # student_set__academic_term_id=pathway_data["academic_term_id"]# using academic term, cause 1 term only can join 1 student set # ) ) students_key_by_id = {s.id: s for s in pathway_students} student_ids = list(students_key_by_id.keys()) # reset every pathway loop, to check got any templete is already fulfil so cant assign more new student ori_pathway_activity_template_ids = [] total_student_set_size = all_pathway_remaining_size[pathway_data['id']] student_set_name_count = "00" # each student set planned size cant more than this value max_student_set_size = pathway_data['min_activity_size'] module_ids = pathway_data['module_ids'] module_ids_set = set(pathway_data["module_ids"] or []) # will return True or False, so if False will stop and continue next all_module_have_template = module_ids_set.issubset(module_have_template) academic_term_data = academic_term_key_by_id[pathway_data['academic_term_id']] student_set_name_prefix = pos_id_name_map[pathway_data['pos_id']]+self.auto_generate_name_seperator if not all_module_have_template: module_no_template = module_ids_set - module_have_template modules_name = TtModule.objects.filter(id__in=module_no_template).values_list("name", flat=True) for display_module_name in modules_name: # show reason of cant generate failed.append({ "module_name": display_module_name, "activity_template_name": None, "activity_count": 0, "reason": __("validation.allocation_error.no_activity_template"), "type": ":no_activity_template" }) has_failed = True # set to 0 first, if not later while loop will have error total_student_set_size = 0 if not has_failed and (total_student_set_size is None or total_student_set_size <= 0): has_failed = True # set to 0 first, if not later while loop will have error total_student_set_size = 0 for activity_template_id in pathway_data['activity_template_ids']: activity_template_data = related_activity_template_data[activity_template_id] failed.append({ "module_name": activity_template_data['module_name'], "activity_template_name": activity_template_data['activity_template_name'], "activity_count": activity_template_data['activity_count'], "reason": __("validation.allocation_error.no_remaining_pathway_size"), "type": "no_remaining_pathway_size" }) while total_student_set_size > 0: student_set_planned_size = int(max_student_set_size or 0) # wont create more than pathway planned size if student_set_planned_size > total_student_set_size: student_set_planned_size = total_student_set_size """ Original query of the ORM SELECT tt_activity.id AS activity_id, tt_activity.activity_template_id as activity_template_id, tt_activity.planned_size - COALESCE(SUM(tt_student_set.planned_size), 0) as remaining_size FROM tt_activity LEFT JOIN tt_student_set_activity ON tt_student_set_activity.activity_id = tt_activity.id LEFT JOIN tt_student_set ON tt_student_set.id = tt_student_set_activity.student_set_id where tt_activity.module_id IN {module_id} GROUP BY tt_activity.id HAVING (tt_activity.planned_size - COALESCE(SUM(tt_student_set.planned_size), 0)) > 0 ORDER by tt_activity.id """ # get activity table id, activity_template_id, remaining_size(planned_size - student_set_activity relation the student_set planned size) activities = ( TtActivity.objects .filter(module_id__in=module_ids) .annotate( remaining_size=F('planned_size') - Coalesce( Sum('ttstudentsetactivity__student_set__planned_size'), Value(0), output_field=IntegerField()) ) .filter(remaining_size__gt=0) .exclude(# exclude jta child and variant child Q(is_jta=1, jta_parent_id__isnull=True) | Q(is_variant=1, variant_parent_id__isnull=False) ) .values('id', 'activity_template_id', 'remaining_size','is_jta','is_variant','jta_parent_id') .order_by('id') ) # check need to generate what planned size for the student set activities = sorted(list(activities), key=itemgetter('activity_template_id')) # after generate the student_set, will assign to those selected_activities # if got activities only generate student_set if activities: selected_activities = [] for key, group in groupby(activities, key=itemgetter('activity_template_id')): group_list = list(group) # assign to those have more remaining_size activity first if allocation_method == "spread": chosen = max(group_list,key=itemgetter('remaining_size')) else: # assign to those have less remaining_size activity first chosen = min(group_list, key=itemgetter('remaining_size')) selected_activities.append(chosen) return_data.append(selected_activities) # check selected_activities all activity_template_id have exact same as activity_template_ids, 1 missing also return error for this pathway available_activity_template_ids = [item["activity_template_id"] for item in selected_activities] pathway_activity_template_ids = pathway_data['activity_template_ids'] # if got missing id, means got activity template fully assign to all activity, so need break the loop and move to next pathway not_found_activity_template_ids = set(pathway_activity_template_ids) - set(available_activity_template_ids) if not_found_activity_template_ids: # return fail reason for not_found_activity_template_id in not_found_activity_template_ids: not_found_data = related_activity_template_data[not_found_activity_template_id] failed.append({ # "module_code": not_found_data['module_code'], "module_name": not_found_data['module_name'], # "activity_template_code": not_found_data['activity_template_code'], "activity_template_name": not_found_data['activity_template_name'], "activity_count": not_found_data['activity_count'], "reason": __("validation.allocation_error.no_available_activity"), "type": "no_activity" }) break # student_set_planned_size wont more than this value max_size_set = min(selected_activities, key=itemgetter('remaining_size')) max_size = max_size_set['remaining_size'] if student_set_planned_size > max_size: student_set_planned_size = max_size # start to loop pathway planned_size, will keep deduct until become 0 then will stop if student_set_planned_size > 0: total_student_set_size = total_student_set_size - student_set_planned_size # handle for name part if missing_student_set_name.get(pathway_data['pos_id']): new_name = missing_student_set_name[pathway_data['pos_id']][0] missing_student_set_name[pathway_data['pos_id']].pop(0) else: last_student_set_name.setdefault(pathway_data['pos_id'], None) last_student_set_name[pathway_data['pos_id']] = get_auto_generate_name(last_student_set_name[pathway_data['pos_id']],student_set_name_prefix, 2) new_name = last_student_set_name[pathway_data['pos_id']] # handle for code part if missing_code: new_code = missing_code[0] missing_code.pop(0) else: last_code = get_auto_generate_code(last_code, self.default_code_prefix, 10) new_code = last_code student_set_name_count = str(int(student_set_name_count) + 1).zfill(len(student_set_name_count)) idata = { "code": new_code, "name": new_name, "desc": "Auto generate by pathway", "pos_id": pathway_data['pos_id'], "department_id": pathway_data['pos_department_id'], "academic_term_id": pathway_data['academic_term_id'], "planned_size": student_set_planned_size, "availability_pattern": academic_term_data.default_availability, "start_preference_pattern": academic_term_data.default_start, "usage_preference_pattern": academic_term_data.default_usage, "status": TtStudentSet.STATUS_TO_CODE['active'] } student_set = TtStudentSet.objects.create(**idata) # allocate student into student set from here for i in range(student_set.planned_size): # if no more student to allocate, stop it if not student_ids: break # get the student id and remove from list student_id = student_ids.pop(0) student = students_key_by_id[student_id] current_student_set_ids = [ss.id for ss in student.student_set.all()] # double check the student not yet join to this student set only will allow the student to join if student_set.id not in current_student_set_ids: student_set_student_relation_idata.append({ "student_set_id": student_set.id, "student_id": student_id, }) resource_map_data = [] resource_map_redis_data = {} for val in weeks_data: resource_map_data.append({ "student_set_id": student_set.id, "week_id": val.id, "pattern": slot_default_pattern['resource_map'], }) # redis week is start from 0, 0 means week 1 redis_week = int(val.week) - 1 resource_map_redis_data[redis_week] = slot_default_pattern['resource_map'] if resource_map_data: resource_map_object = [TtStudentSetResourceMap(**data) for data in resource_map_data] TtStudentSetResourceMap.objects.bulk_create(resource_map_object) student_set_generated_count += 1 # data need insert to redis redis_data["insert"].setdefault(redis_student_set_table, []).append({ "id": student_set.id, "code": student_set.code, "name": student_set.name, "department_id": student_set.department_id, "constraint_profile": [], "pos_id": student_set.pos_id, "academic_term_id": student_set.academic_term_id, "planned_size": student_set.planned_size, "module_id": module_ids or [], "availability_id": 0,# schedule engine request if no preset, cant put null, need be 0 "availability_pattern": student_set.availability_pattern, "start_preference_id": 0,# schedule engine request if no preset, cant put null, need be 0 "start_preference_pattern": student_set.start_preference_pattern, "usage_preference_id": 0,# schedule engine request if no preset, cant put null, need be 0 "usage_preference_pattern": student_set.usage_preference_pattern, "resource_map": resource_map_redis_data, "allocated_activity_ids": [],# when scheduled activity only will add to this column "student_ids": [], }) # insert student_set_module relation student_set_module_relation_idata = [] kafka_student_set_module = module_ids for module_id in module_ids: student_set_module_relation_data = { "student_set_id": student_set.id, "module_id": module_id, } student_set_module_relation_idata.append(student_set_module_relation_data) # save module_ids if student_set_module_relation_idata: # convert dicts -> model instances, if not object cant used bulk_create student_set_relation_objects = [TtStudentSetModule(**data) for data in student_set_module_relation_idata] # this transaction.atomic() is when got error will rollback with transaction.atomic(): for i in range(0, len(student_set_relation_objects), batch_size): TtStudentSetModule.objects.bulk_create( student_set_relation_objects[i:i + batch_size], batch_size=batch_size ) # insert student_set_activity relation student_set_activity_relation_idata = [] kafka_student_set_activity = set() for activity in selected_activities: # check is variant, is jta and is jta + variant and update student set also # when activity is variant if activity.get("is_variant"): variant_child_activities = TtActivity.objects.filter(variant_parent_id=activity["id"]) for variant_child_activity in variant_child_activities: kafka_student_set_activity.add(variant_child_activity.id) affected_activity_ids.add(variant_child_activity.id) student_set_activity_relation_idata.append({ "student_set_id": student_set.id, "activity_id": variant_child_activity.id, }) # when activity is JTA if activity.get("is_jta") and activity.get("jta_parent_id"): jta_child_activity = TtActivity.objects.filter(id=activity["jta_parent_id"]).first() if jta_child_activity: kafka_student_set_activity.add(jta_child_activity.id) affected_activity_ids.add(jta_child_activity.id) student_set_activity_relation_idata.append({ "student_set_id": student_set.id, "activity_id": jta_child_activity.id, }) # when activity is JTA Variant if jta_child_activity.is_variant: jta_variant_child_activities = TtActivity.objects.filter(variant_parent_id=jta_child_activity.id) for jta_variant_child_activity in jta_variant_child_activities: kafka_student_set_activity.add(jta_variant_child_activity.id) affected_activity_ids.add(jta_variant_child_activity.id) student_set_activity_relation_idata.append({ "student_set_id": student_set.id, "activity_id": jta_variant_child_activity.id, }) student_set_activity_relation_data = { "student_set_id": student_set.id, "activity_id": activity['id'], } kafka_student_set_activity.add(activity['id']) affected_activity_ids.add(activity['id']) student_set_activity_relation_idata.append(student_set_activity_relation_data) if student_set_activity_relation_idata: # convert dicts -> model instances, if not object cant used bulk_create student_set_activity_relation_objects = [TtStudentSetActivity(**data) for data in student_set_activity_relation_idata] # this transaction.atomic() is when got error will rollback with transaction.atomic(): for i in range(0, len(student_set_activity_relation_objects), batch_size): TtStudentSetActivity.objects.bulk_create( student_set_activity_relation_objects[i:i + batch_size], batch_size=batch_size, ignore_conflicts=True ) # kafka add kafka_data = model_to_dict(student_set) if kafka_student_set_module: kafka_data['module'] = kafka_student_set_module if kafka_student_set_activity: kafka_data['activity'] = list(kafka_student_set_activity) #remove any None key kafka_data.pop("resource_map", None) kafka_data = {k: v for k, v in kafka_data.items() if v is not None} kafka_student_set.append(kafka_data) else: # if no student_set_planned_size, can exit the loop total_student_set_size = 0 else: # exit loop if no activities found for failed_module_id in module_ids: not_found_data = related_module_data[failed_module_id] failed.append({ "module_name": not_found_data['module_name'], "activity_template_name": None, "activity_count": 0, "reason": __("validation.allocation_error.no_available_activity"), "type": "no_activity" }) total_student_set_size = 0 # if got new student set was generated, will insert 1 time student set student relation first # if got remaining student not yet allocate, will continue find available student set to allocate if student_set_student_relation_idata: allocated_student_count += len(student_set_student_relation_idata) # convert dicts -> model instances, if not object cant used bulk_create student_set_student_relation_objects = [TtStudentSetStudent(**data) for data in student_set_student_relation_idata] with transaction.atomic(): TtStudentSetStudent.objects.bulk_create( student_set_student_relation_objects, batch_size=batch_size, ignore_conflicts=True ) # if still got student not yet allocate in this pathway, double check got other available student set to allocate or not if student_ids: student_ids, student_set_student_relation_idata_2 = allocate_student_to_old_student_set(student_ids,pathway_data,students_key_by_id) allocated_student_count += len(student_set_student_relation_idata_2) # group tgt, later will based on student_set_student_relation_idata to generate kafka data student_set_student_relation_idata.extend(student_set_student_relation_idata_2) # generate student allocate data for kafka microservices if student_set_student_relation_idata: for val in student_set_student_relation_idata: # recalculate_student_ids will used for calculate resource map for student recalculate_student_ids.add(val["student_id"]) if not kafka_student_allocate_data.get(val['student_set_id']): kafka_student_allocate_data[val['student_set_id']] = { "student_set_id": val['student_set_id'], "student_ids": [] } kafka_student_allocate_data[val['student_set_id']]["student_ids"].append(val["student_id"]) # if got msg same module name and type, remove if failed: seen = set() unique_failed = [] for item in failed: key = (item["module_name"], item.get("activity_template_name"), item["type"]) if key not in seen: seen.add(key) unique_failed.append(item) failed = unique_failed if affected_activity_ids: scheduled_student_set_ids = set() student_set_with_affected_week = {} activity_data = TtActivity.objects.filter(id__in=affected_activity_ids).prefetch_related("student_set","week","week_pattern__week") for a in activity_data: activity_student_set_ids = [ss.id for ss in a.student_set.all()] if a.scheduled: if a.week_pattern: week_ids = [aw.id for aw in a.week_pattern.week.all()] else: week_ids = [aw.id for aw in a.week.all()] for student_set_id in activity_student_set_ids: student_set_with_affected_week.setdefault(student_set_id, set()).update(week_ids) scheduled_student_set_ids.update(activity_student_set_ids) # in redis this column call student_sets redis_data["update"].setdefault(redis_activity_table, []).append({"id":a.id,"student_sets": activity_student_set_ids,}) student_set_with_affected_week = {k: v for k, v in student_set_with_affected_week.items() if v} # since got update student set, this helper already cater if got update student set, no need pass in student details also will update student resource map # even after this line will call again same function to update student resourcemap, but this time after update student set, will still update related student resource map, just prevent got any missing redis_data = helper_recalculate_and_get_redis_resource_map( redis_data=redis_data, student_set_with_affected_week=student_set_with_affected_week, slot_per_week=slot_per_week, ) if kafka_student_allocate_data: for key,val in kafka_student_allocate_data.items(): redis_data["update"].setdefault(redis_student_set_table, []).append({"id":key,"student_ids": val["student_ids"],}) if recalculate_student_ids: students_model = TtStudent.objects.filter(id__in=list(recalculate_student_ids)).prefetch_related("student_set") for student_model in students_model: student_set_ids = list({ss.id for ss in student_model.student_set.all()}) redis_data["update"].setdefault(redis_student_table, []).append({"id":student_model.id,"student_set_ids": student_set_ids,}) # because have possible to allocate student into old student set that not generated by this time api call, so having this function to handle if recalculate_week_ids and recalculate_student_ids: redis_data = helper_recalculate_and_get_redis_resource_map( redis_data=redis_data, student_ids=list(recalculate_student_ids), week_ids=list(recalculate_week_ids), slot_per_week=slot_per_week, ) if redis_data['insert'] or redis_data['update']: bulk_sync_to_redis(redis_client, redis_data) #kafka push method = "allocation" kafka_topic = self.kafka_config['MICROSERVICES_TT_TOPIC'] if kafka_topic and (kafka_student_set or kafka_student_allocate_data): kafka_request_data = { "session_id": request.user.name, "student_set": kafka_student_set, "student_allocate": list(kafka_student_allocate_data.values()) # using list to remove the key } send_request(kafka_topic, kafka_request_data, None, method) # insert audit trail and details # can insert the parent first, then only insert details audit_trail = AuditTrail.objects.create( user_id=request.user.id, type=self.audit_type, ip_address=get_ip(request) ) generated_student_set_ids = [item["id"] for item in kafka_student_set] remark_param = { # "code":student_set.code, "name": ", ".join(pathway_names.keys()), } old_new_data["new_data"] = { "method": allocation_method, "allocated_student_count": allocated_student_count, "generated_student_set_ids": generated_student_set_ids, "generated_count": student_set_generated_count, "failed_count": len(failed), "pathway_ids": ids, } # call to a function for insert details, so in future if want change to used worker, can modify in function only AuditTrailDetails.custom_insert( audit_trail=audit_trail, action=action, remark_param=remark_param, new_data=old_new_data['new_data'], ) response = { "success": { "student_allocated": allocated_student_count, "student_set_generated": student_set_generated_count, "failed_allocated": 0, }, "failed": failed } return self.api_response(data=response) except ValidationError as e: first_message = e.detail['error'] errors = e.detail['errors'] return self.api_response(error=first_message,errors=errors,code=status.HTTP_400_BAD_REQUEST) except Exception as e: e_details = get_exception_detail(e) log_critical_error(user_id=None,descr=e_details['descr'],url=e_details['url'],trace=e_details['trace']) return self.api_response(error=__("message.internal_server_error"),code=status.HTTP_500_INTERNAL_SERVER_ERROR) def get_related_pathway(ids): pathways = ( TtPathway.objects .filter( id__in=ids, status=TtPathway.STATUS_TO_CODE['active'] ) .annotate( module_ids=ArrayAgg( "ttpathwayposmodulegroupmodule__pos_module_group_module__module_id", distinct=True ), min_activity_size=Min( "ttpathwayposmodulegroupmodule__pos_module_group_module__module__ttactivity__planned_size" ), pos_name=F("pos__name"), pos_department_id=F("pos__department_id"), activity_template_ids=ArrayAgg( "ttpathwayposmodulegroupmodule__pos_module_group_module__module__ttactivitytemplate__id", # "ttpathwayposmodulegroupmodule__pos_module_group_module__module__ttactivity__activity_template_id", distinct=True, filter=Q(ttpathwayposmodulegroupmodule__pos_module_group_module__module__ttactivitytemplate__id__isnull=False) ), ) .values("id", "planned_size", "module_ids", "min_activity_size", "pos_name","pos_department_id", "pos_id", "academic_term_id","activity_template_ids") ) if not pathways: return [] pos_ids = {p["pos_id"] for p in pathways if p["pos_id"]} all_module_ids = {m_id for p in pathways for m_id in (p["module_ids"] or []) if m_id} # get all related student set by pos_ids and module_ids, later will check which is same as the pathway student_sets = ( TtStudentSet.objects.filter( pos_id__in=pos_ids, module__id__in=all_module_ids # Spans M2M relation ) .values("id", "pos_id") .annotate( ss_module_ids=ArrayAgg("module__id", distinct=True) ) ) pos_to_student_sets = {} # form data key by pos_id with value student set id and module ids for ss in student_sets: pos_id = ss["pos_id"] ss_id = ss["id"] # convert to set(), so later can direct compare with the module_ids set in pathway ss_modules = set(ss["ss_module_ids"] or []) pos_to_student_sets.setdefault(pos_id, []).append({ "student_set_id": ss_id, "module_ids_set": ss_modules }) for p in pathways: p_pos_id = p["pos_id"] # convert module_ids to set for doing compare checking p_module_set = set(p["module_ids"] or []) student_set_ids = [] # if pathway pos_id isset of pos_to_student_sets and pathway module_ids is not empty, will start to check student set is belong to which if p_pos_id in pos_to_student_sets and p_module_set: for ss_val in pos_to_student_sets[p_pos_id]: if ss_val["module_ids_set"] == p_module_set: student_set_ids.append(ss_val["student_set_id"]) p["student_set_ids"] = student_set_ids return pathways def calculate_pathway_remaining_size(pathway_data): """ return data like pathway_id: remaining_size eg. [ 1: 100, 2: 150 ] """ # get this 2 id to reduce the result get from db pos_ids = list(pathway_data.values_list("pos_id",flat=True)) academic_term_ids = list(pathway_data.values_list("academic_term_id",flat=True)) student_sets = { s["id"]: s for s in ( TtStudentSet.objects .filter(pos_id__in=pos_ids,academic_term_id__in=academic_term_ids) .annotate( module_ids=ArrayAgg("module__id", distinct=True) ) .values("id", "planned_size", "module_ids") ) } # Build the final dict: pathway_id → remaining_size pathway_remaining = {} #compare pathway-data module_ids is same as student_set module_ids, only count the planned_size as used size for p in pathway_data: total_size = sum( s["planned_size"] for s in student_sets.values() if set(s["module_ids"]) == set(p["module_ids"]) ) pathway_remaining[p["id"]] = p["planned_size"] - total_size return pathway_remaining def allocate_student_to_old_student_set(student_ids,pathway_data,students_key_by_id): """ this function is allocate student into those student set not created in this time """ student_set_student_relation_idata_2 = [] target_module_ids = set(pathway_data['module_ids']) target_count = len(target_module_ids) pathway_student_set_query = ( TtStudentSet.objects .filter( pos_id=pathway_data['pos_id'], academic_term_id=pathway_data['academic_term_id'], ) .annotate( module_count=Count("module", distinct=True), student_count=Count("student", distinct=True), ) .filter( student_count__lt=F("planned_size"), module_count=target_count ) ) for m_id in target_module_ids: pathway_student_set_query = pathway_student_set_query.filter(module__id=m_id) pathway_student_sets = {s.id: s for s in pathway_student_set_query} for ss_id, ss_data in pathway_student_sets.items(): planned_size = ss_data.planned_size or 0 remaining_size = planned_size - ss_data.student_count if remaining_size > 0: for i in range(remaining_size): # if no more student to allocate, stop it if not student_ids: break # get the student id and remove from list student_id = student_ids.pop(0) student = students_key_by_id[student_id] current_student_set_ids = [ss.id for ss in student.student_set.all()] # double check the student not yet join to this student set only will allow the student to join if ss_data.id not in current_student_set_ids: student_set_student_relation_idata_2.append({ "student_set_id": ss_data.id, "student_id": student_id, }) if student_set_student_relation_idata_2: # convert dicts -> model instances, if not object cant used bulk_create student_set_student_relation_objects_2 = [TtStudentSetStudent(**data) for data in student_set_student_relation_idata_2] with transaction.atomic(): TtStudentSetStudent.objects.bulk_create( student_set_student_relation_objects_2, batch_size=500, ignore_conflicts=True ) return student_ids,student_set_student_relation_idata_2