from datetime import datetime, date, time from django.core.management.base import BaseCommand from django.conf import settings from django.db.models import Q from django.forms import model_to_dict from api.models import TtStaff, TtLocation, TtStudentSet, TtActivityTemplate, TtActivity, TtDepartment, TtZone, \ TtAcademicTerm, TtWeek, TtTag, TtUsagePreference, TtStartPreference, TtAvailability, TtSuitability, TtWeekPattern, \ TtModuleGroup, TtPos, TtModule, TtPosModuleGroup, TtActivityType, TtStudent from api.utils import convert_to_redis_week, bulk_sync_to_redis, generate_kafka_data from backend.kafka import send_request from backend.redis_client import redis_client class Command(BaseCommand): def generate_kafka_data(self, all_data, relation_fields=None,exclude=None): if not all_data: return {} if relation_fields is None: relation_fields = [] if exclude is None: exclude = [] shared_fields = [] exclude.extend(["pathway","resource_map", "created_by", "updated_by"]) for data in all_data: format_data = model_to_dict(data,exclude=exclude) for key, value in format_data.items(): if isinstance(value, (datetime, date, time)): format_data[key] = value.isoformat() if relation_fields: for field in relation_fields: if field == "tag": format_data[field] = [tag.tag.id for tag in data.tag.all()] else: format_data[field] = [m2m.id for m2m in getattr(data, field).all()] shared_fields.append(format_data) return shared_fields def handle(self, *args, **options): kafka_topic = settings.KAFKA_CONFIG['MICROSERVICES_TT_TOPIC'] # academic_term method = "academic_term_create" new_data = TtAcademicTerm.objects.all().prefetch_related('week').order_by("id") relation_fields = [ "week" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) weeks = TtWeek.objects.all() week_data = [ { "id": week.id, "week": week.week, "start_date": week.start_date.strftime("%Y-%m-%d") } for week in weeks ] kafka_request_data = { "session_id": "", "academic_term": kafka_data, "week": week_data, } send_request(kafka_topic, kafka_request_data, None, method) # tag method = "tag_create" new_data = TtTag.objects.all().order_by("id") kafka_data = self.generate_kafka_data(new_data) kafka_request_data = { "session_id": "", "tag": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # zone method = "zone_create" new_data = TtZone.objects.all().prefetch_related('tag__tag').order_by("id") relation_fields = [ "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "zone": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) #deparmtnet method = "department_create" new_data = TtDepartment.objects.all().prefetch_related('tag__tag').order_by("id") relation_fields = [ "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "department": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # if got problem, might need add zone update and department update at here to update department and zone id in their table # usage_preference method = "usage_preference_create" new_data = TtUsagePreference.objects.all().prefetch_related('tag__tag').order_by("id") relation_fields = [ "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "usage_preference": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # start_preference method = "start_preference_create" new_data = TtStartPreference.objects.all().prefetch_related('tag__tag').order_by("id") relation_fields = [ "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "start_preference": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # availability method = "availability_create" new_data = TtAvailability.objects.all().prefetch_related('tag__tag').order_by("id") relation_fields = [ "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "availability": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # suitability method = "suitability_create" new_data = TtSuitability.objects.all().prefetch_related('staff','location','tag__tag').order_by("id") relation_fields = [ "staff", "location", "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "suitability": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # staff method = "staff_create" new_data = TtStaff.objects.all().prefetch_related("suitability","avoid_concurrency","shared_with_department","constraint_profile",'tag__tag').order_by("id") relation_fields = [ "suitability", "avoid_concurrency", "shared_with_department", "constraint_profile", "tag", ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "staff": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # location method = "location_create" new_data = TtLocation.objects.all().prefetch_related("suitability", "avoid_concurrency", "shared_with_department","constraint_profile", 'tag__tag').order_by("id") relation_fields = [ "suitability", "avoid_concurrency", "shared_with_department", "constraint_profile", "tag", ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "location": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # week_pattern method = "week_pattern_create" new_data = TtWeekPattern.objects.all().prefetch_related('week','tag__tag').order_by("id") relation_fields = [ "week", "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "week_pattern": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # module_group method = "module_group_create" new_data = TtModuleGroup.objects.all().order_by("id") relation_fields = [] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "module_group": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # pos method = "pos_create" new_data = TtPos.objects.all().prefetch_related('module_group','tag__tag').order_by("id") relation_fields = [ "module_group", "student", "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "pos": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # module method = "module_create" new_data = TtModule.objects.all().prefetch_related('week','tag__tag').order_by("id") relation_fields = [ "week", "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "module": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # module_group_module method = "pos_update_module" pos_module_groups = TtPosModuleGroup.objects.all().prefetch_related("module").order_by("id") kafka_data = [ { "pos_module_group_id": pos_module_group.id, "pos_id": pos_module_group.pos_id, "module_group": pos_module_group.module_group_id, "module_id": list(pos_module_group_module.id for pos_module_group_module in pos_module_group.module.all()) } for pos_module_group in pos_module_groups ] kafka_request_data = { "session_id": "", "modules": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # activity_type method = "activity_type_create" new_data = TtActivityType.objects.all().prefetch_related('tag__tag').order_by("id") relation_fields = [ "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "activity_type": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # activity template method = "activity_template_create" new_data = TtActivityTemplate.objects.all().prefetch_related('staff','staff_suitability','location','location_suitability','week','tag__tag').order_by("id") relation_fields = [ "staff", "staff_suitability", "location", "location_suitability", "week", "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields) kafka_request_data = { "session_id": "", "activity_template": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # activity exclude jta parent and variant child first, later only handle for jta parent and variant child method = "activity_create" new_data = TtActivity.objects.filter( is_booking=0 ).exclude( Q(is_jta=1, jta_parent_id__isnull=False) | Q(is_variant=1, variant_parent_id__isnull=False) ).prefetch_related( 'staff_preset', 'staff_suitability', 'staff', 'location', 'location_preset', 'location_suitability', 'week', 'tag__tag' ).order_by("id") relation_fields = [ "staff_preset", "staff_suitability", "location_preset", "location_suitability", "staff", "location", "week", "tag", ] kafka_data = self.generate_kafka_data(new_data, relation_fields,["student_set","sequencing"]) kafka_request_data = { "session_id": "", "activity": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # activity for jta parent and variant child only, cause if insert these data same time with normal activity, will have error method = "activity_create" new_data = TtActivity.objects.filter( Q(is_jta=1, jta_parent_id__isnull=False) | Q(is_variant=1, variant_parent_id__isnull=False), is_booking=0 ).prefetch_related( 'staff_preset', 'staff_suitability', 'staff', 'location', 'location_preset', 'location_suitability', 'week', 'tag__tag' ).order_by("id") relation_fields = [ "staff_preset", "staff_suitability", "location_preset", "location_suitability", "staff", "location", "week", "tag", ] kafka_data = self.generate_kafka_data(new_data, relation_fields, ["student_set", "sequencing"]) kafka_request_data = { "session_id": "", "activity": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) method = "booking_create" new_data = TtActivity.objects.filter(is_booking=1).prefetch_related('staff_preset', 'staff_suitability','staff', 'location', 'location_preset','location_suitability', 'week','tag__tag').order_by("id") relation_fields = [ "staff_preset", "staff_suitability", "location_preset", "location_suitability", "staff", "location", "week", "tag", ] kafka_data = self.generate_kafka_data(new_data, relation_fields, ["student_set", "sequencing"]) kafka_request_data = { "session_id": "", "activity": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # student_set method = "student_set_create" new_data = TtStudentSet.objects.all().prefetch_related( # "constraint_profile", "activity", "module", "tag__tag" ).order_by("id") relation_fields = [ "constraint_profile", "activity", "module", # "student", "tag" ] kafka_data = self.generate_kafka_data(new_data, relation_fields,["student"]) kafka_request_data = { "session_id": "", "student_set": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method) # student method = "student_create" new_data = TtStudent.objects.all().prefetch_related( "academic_term", "pos", "pos_module__pos_module_group", "student_set", "tag__tag", ).order_by("id") relation_fields = [ "academic_term", "pos", "pos_module", "student_set", "tag", ] kafka_data = self.generate_kafka_data(new_data, relation_fields) for student_data, student in zip(kafka_data, new_data): student_data["pos_module"] = [ { "pos_module_group_module_id": pos_module.id, "pos_id": pos_module.pos_module_group.pos_id, "module_group_id": pos_module.pos_module_group.module_group_id, "module_id": pos_module.module_id, } for pos_module in student.pos_module.all() ] kafka_request_data = { "session_id": "", "student": kafka_data, } send_request(kafka_topic, kafka_request_data, None, method)