I'm having a trouble while creating a schema through python classes and it is not generating through subclass function but when I try the same in outside of the class, and it is working fine.
For more information about the issue, I have copied my sample code, input data, output information as well. Please let me know.
from datetime import datetime from datetime import date dt=datetime.strptime('2019-11-23', "%Y-%m-%d").date() print(dt) endpoint = 'update' print(endpoint) import json from decimal import Decimal from pyspark.sql.types import StructType, ArrayType, StructField, BooleanType, DecimalType, _infer_type def create_schema(prototype): if isinstance(prototype, dict): print("I am dictionary") return StructType([StructField(k, create_schema(v), True) for k, v in sorted(prototype.items())]) elif isinstance(prototype, list): print("I am list") return ArrayType(create_schema(prototype[0]), True) elif isinstance(prototype, Decimal): print("I am decimal") return DecimalType(precision=38, scale=2) elif isinstance(prototype, bool): print("I am boolean") return BooleanType() else: print("I am nothing") return _infer_type(prototype) def prototype(string_dt): if endpoint == 'update': print("I called prototype for update successfully") result = 'schema_' + endpoint + '_' + string_dt print(result) return result else: endpoint == 'create' print("I called prototype for create successfully") result = 'schema_' + endpoint + '_' + string_dt print(result) return result mapping = { (date(2019, 11, 1), date(2020, 2, 28)): { "schema": prototype } } class Router: def __init__(self, mapping, dt, **kwargs): self.mapping = mapping ranges = mapping.keys() for k, v in kwargs.items(): setattr(self, k, v) for drange in ranges: if drange[0] <= dt <= drange[1]: self.range = drange dt1 = drange[0] dt1 = datetime.strptime(str(dt1), '%Y-%m-%d').strftime('%Y%m%d') dt2 = drange[1] dt2 = datetime.strptime(str(dt2), '%Y-%m-%d').strftime('%Y%m%d') self.string_date = dt1 + '_' + dt2 print(self.string_date) self.routed = self.route() self.retVal = self.__call__() #print([self.retVal]) #print('I found which func you need to run!') def route(self): print("I called route successfully") #print(self.mapping[self.range][self.type_key]) return self.mapping[self.range][self.type_key](self.string_date) @property def type_key(self): raise NotImplementedError class SchemaRouter(Router): @property def type_key(self): return 'schema' def __call__(self): print('You are calling schema') print(self.routed) output_schema = create_schema('self.routed') print(output_schema) return output_schema #return create_schema(self.routed) class FunctionRouter(Router): @property def type_key(self): return 'function' def route(self): return self.mapping[self.range][self.type_key][self.table] def __call__(self, *args, **kwargs): return self.routed(*args, **kwargs)
''' Input data ''' schema_update_20191101_20200228 = { "robotIds": [ "11111111" ], "robotRecord": "GAMMA", "payingrobotId": "11111111", "robotType": "smart", "robotBeginDate": "2019-08-26", "robotEndDate": "2019-11-23", "robotBeginDate": "2019-08-26", "robotEndDate": "2019-11-23", "robotColl": "ABC", "robotAmt": Decimal(2.8), "robotFeeCredit": Decimal(0.59), "status": "XXXXXXX", "statusReason": "XXXXXXXXXXXXX.", "dupeId": "XXXXXXXXXXXXXXXXXXX", "robotDate": "2019-11-24 08:00:19.531", "robotendDate": "2019-11-24 08:00:19.531" }
''' Actual output:
obj20 = SchemaRouter(mapping, dt) 20191101_20200228 I called route successfully I called prototype for update successfully schema_update_20191101_20200228 You are calling schema schema_update_20191101_20200228 I am nothing StringType '''
''' Expected output: If I run the schema without SchemaRouter class, I'm getting the output_schema result. output_schema = create_schema(20191101_20200228) '''
'''
https://stackoverflow.com/questions/65546461/passing-structname-as-the-parameter-to-a-function-in-python January 03, 2021 at 11:50AMprint(output_schema) StructType(List(StructField(robotIds,ArrayType(StringType,true),true),StructField(robotRecordKeeper,StringType,true),StructField(createdDate,StringType,true),StructField(eventType,StringType,true),StructField(robotCredit,DecimalType(38,2),true),StructField(robotAmount,DecimalType(38,2),true),StructField(robotCollectionEntity,StringType,true),StructField(robotBeginDate,StringType,true),StructField(robotEndDate,StringType,true),StructField(invoiceId,StringType,true),StructField(lastUpdatedDate,StringType,true),StructField(robotStatusBeginDate,StringType,true),StructField(robotStatusEndDate,StringType,true),StructField(payingrobotId,StringType,true),StructField(status,StringType,true),StructField(statusReason,StringType,true))) '''
没有评论:
发表评论