PKKNAspyulgresample/__init__.py"""pyulgresample. A package that resamples ulog topics into one pandas dataframe. The package also contains convenient functions to exctact addional information from the ulog topics. """ __version__ = "0.0.2" PK!\N pyulgresample/loginfo.py"""Get general ulog info.""" import pyulog import numpy as np import datetime import warnings def get_ulog(filepath, topics=None): """Read a ulg file from the given filepath and return it as a ulog structure. It can be that sometimes, topics are missing. Thus, check if the required topic are available in the ulog file. Arguments: filepath -- absoulte path to the .ulg file topics -- list of required topics """ if topics: ulog = pyulog.ULog(filepath, topics) tmp = topics.copy() for topic in ulog.data_list: if topic.name in tmp: idx = tmp.index(topic.name) tmp.pop(idx) if len(tmp) > 0: warnings.warn( "The following topics do not exist: \n {0}".format(tmp) ) else: ulog = pyulog.ULog(filepath) if not ulog.data_list: warnings.warn("No topics present.") return ulog def mu2hms(musecond): """convert microsecond to hours:min:second (string).""" m1, s1 = divmod(int(musecond / 1e6), 60) h1, m1 = divmod(m1, 60) return "{:d}:{:02d}:{:02d}".format(h1, m1, s1) def get_starttime(ulog): """Recover the start time stored in the ulog structure. Arguments: ulog -- messages stored in ulog structure """ return mu2hms(ulog.start_timestamp) def get_duration(ulog): """Compute the duration for which data was logged. Arguments: ulog -- messages stored in ulog structure """ return mu2hms(ulog.last_timestamp - ulog.start_timestamp) def get_param(ulog, parameter_name, default): """Recover a parameter from the ulog structure. Arguments: ulog -- messages stored in ulog structure parameter_name -- name of the parameter that should be recovered default -- default value that will be returned if the parameter is not available """ if parameter_name in ulog.initial_parameters.keys(): return ulog.initial_parameters[parameter_name] else: return default def add_param(dfUlg, parameter_name): """add a parameter from the ulog structure to the dataframe. If parameters have changed, update them in the dataframe. Arguments: ulog -- messages stored in ulog structure parameter_name -- name of the parameter that should be recovered dataframe -- pandas dataframe which contains all messages of the required topics """ dfUlg.df[parameter_name] = get_param(dfUlg.ulog, parameter_name, 0) if len(dfUlg.ulog.changed_parameters) > 0 and ( parameter_name in [x[1] for x in dfUlg.ulog.changed_parameters] ): dfUlg.df[parameter_name] = np.nan for time, name, value in dfUlg.ulog.changed_parameters: if name == parameter_name: dfUlg.df.loc[ (dfUlg.df.timestamp <= time) & (np.isnan(dfUlg.df[parameter_name])), parameter_name, ] = value dfUlg.df[parameter_name].fillna(method="ffill", inplace=True) PKKN]kXpyulgresample/mathpandas.py"""Pandas series / dataframe manipulation.""" import pandas as pd import transforms3d.quaternions as quat import transforms3d.taitbryan as tf import numpy as np import utm def combine_names(msg_name, new_name): """combine msg-name with a new name.""" if msg_name: return "{0}_{1}".format(msg_name, new_name) else: return new_name def get_series_quat2euler(q0, q1, q2, q3, msg_name=""): """Given pandas series q0-q4, compute series roll, pitch, yaw. Arguments: q0-q4 -- quaternion entries Keyword arguments: msg_name -- name of the message for which the euler angles should be computed (default "") """ yaw, pitch, roll = np.array( [ tf.quat2euler([q0i, q1i, q2i, q3i]) for q0i, q1i, q2i, q3i in zip(q0, q1, q2, q3) ] ).T yaw = pd.Series( name=combine_names(msg_name, "yaw"), data=yaw, index=q0.index ) pitch = pd.Series( name=combine_names(msg_name, "pitch"), data=pitch, index=q0.index ) roll = pd.Series( name=combine_names(msg_name, "roll"), data=roll, index=q0.index ) return roll, pitch, yaw def angle_wrap_pi(x): """wrap angle between -pi and pi. Arguments: x -- angle to be wrapped """ return np.arcsin(np.sin(x)) def get_series_quatrot(x, y, z, q0, q1, q2, q3, msg_name=""): """Given pandas series x-z and quaternion q0-q4, compute rotated vector x_r, y_r, z_r. Arguments: x,y,z -- vector to be rotated q0-q4 -- quaternion entries. The vector is being rotated with this quaternion Keyword arguments: rot_name -- name of the rotation """ vec = np.array( [ quat.rotate_vector([xi, yi, zi], [q0i, q1i, q2i, q3i]) for xi, yi, zi, q0i, q1i, q2i, q3i in zip(x, y, z, q0, q1, q2, q3) ] ) x_r = pd.Series( name=combine_names(msg_name, "x"), data=vec[:, 0], index=x.index ) y_r = pd.Series( name=combine_names(msg_name, "y"), data=vec[:, 1], index=y.index ) z_r = pd.Series( name=combine_names(msg_name, "z"), data=vec[:, 2], index=z.index ) return x_r, y_r, z_r def get_series_quatrot_inverse(x, y, z, q0, q1, q2, q3, msg_name=""): """Given pandas series x-z and quaternion q0-q4, compute reversed rotated vector x_r, y_r, z_r. Arguments: x,y,z -- vector to be rotated q0-q4 -- quaternion entries. The vector is being rotated with the inverse of that quaternion Keyword arguments: rot_name -- name of the rotation """ return get_series_quatrot(x, y, z, q0, -q1, -q2, -q3, msg_name) def get_series_dot(x0, y0, z0, x1, y1, z1, msg_name=""): """Given pandas series x0-z0 and x1-z1, compute dot product. Arguments: x0, y0, z0 -- first vector x1, y1, z1 -- second vector Keyword Arguments: dotname -- name of the newly created data (default "") """ dot = np.array( [ np.dot([x0i, y0i, z0i], [x1i, y1i, z1i]) for x0i, y0i, z0i, x1i, y1i, z1i in zip(x0, y0, z0, x1, y1, z1) ] ) return pd.Series( name=combine_names(msg_name, "dot"), data=dot, index=x0.index ) def get_series_norm_2d(x0, y0, msg_name=""): """Given pandas series x0-y0, compute norm. Arguments: x0 -- first pandas series y0 -- second pandas series Keyword Arguments: dotname -- name of the newly created data (default "") """ norm = np.array( [np.linalg.norm([x0i, y0i], axis=0) for x0i, y0i in zip(x0, y0)] ) return pd.Series( name=combine_names(msg_name, "norm"), data=norm, index=x0.index ) def get_series_utm(lat, lon, msg_name=""): """Given pandas series lat/lon in degrees, compute UTM easting/northing/zone. Arguments: lat -- latitude lon -- longitude Keyword Arguments: msg_name -- name of the newly created data (default "") """ easting, northing, zone, letter = np.array( [utm.from_latlon(lati, loni) for lati, loni in zip(lat, lon)] ).T easting = pd.Series( name=combine_names(msg_name, "easting"), data=easting.astype(np.float), index=lat.index, ) northing = pd.Series( name=combine_names(msg_name, "northing"), data=northing.astype(np.float), index=lat.index, ) zone = pd.Series( name=combine_names(msg_name, "zone"), data=zone.astype(np.float), index=lat.index, ) return easting, northing, zone def get_z_axis_from_attitude(q0, q1, q2, q3, msg_name=""): """Compute the desired body z axis in world coordinate system. Arguments: q0 -- pandas time series quaternion element 0 q1 -- pandas time series queternion element 1 q2 -- pandas time series quaternion element 2 q3 -- pandas time series quaternion element 3 Keyword Arguments: msg_name -- name of the newly created data (default "") """ x = pd.Series( np.zeros(q0.shape[0]), index=q0.index, name=combine_names(msg_name, "z_axis_x"), ) y = pd.Series( np.zeros(q0.shape[0]), index=q0.index, name=combine_names(msg_name, "z_axis_y"), ) z = pd.Series( np.ones(q0.shape[0]), index=q0.index, name=combine_names(msg_name, "z_axis_z"), ) x, y, z = get_series_quatrot(x, y, z, q0, q1, q2, q3) return x, y, z def get_tilt_from_attitude(q0, q1, q2, q3, msg_name=""): """Compute desired tilt angle and add it to the dataframe. Arguments: q0 -- pandas time series quaternion element 0 q1 -- pandas time series queternion element 1 q2 -- pandas time series quaternion element 2 q3 -- pandas time series quaternion element 3 Keyword Arguments: msg_name -- name of the newly created data (default "") """ z_x, z_y, z_z = get_z_axis_from_attitude(q0, q1, q2, q3) x = pd.Series(np.zeros(q0.shape[0]), index=q0.index, name="x") y = pd.Series(np.zeros(q0.shape[0]), index=q0.index, name="y") z = pd.Series(np.ones(q0.shape[0]), index=q0.index, name="z") tilt = get_series_dot(x, y, z, z_x, z_y, z_z, msg_name) tilt.where( tilt < 1, 1, inplace=True ) # ensure that angle 1 is never exceeded return tilt.apply(np.arccos) def get_normalize_2d_vector(x, y, msg_name=""): """Normalize 2D vector. Arguments: x -- pandas time series x component of vector y -- pandas time series y component of vector Keyworkd Arguments: msg_name -- name of the newly created data (default "") """ norm = get_series_norm_2d(x, y, msg_name=msg_name) norm[norm <= np.finfo(np.float64).eps] = np.finfo(np.float64).eps x_n = x / norm y_n = y / norm return x_n, y_n def get_heading_from_2d_vector(north, east, msg_name=""): """Compute the heading (0 heading = North) from a 2D vector. 0 Heading = north is only true if north/east is aligned with GPS coordinate system. Arguments: x -- pandas time series x component of vector y -- pandas time series y component of vector Keyworkd Arguments: msg_name -- name of the newly created data (default "") """ x_n, y_n = get_normalize_2d_vector(north, east, msg_name) return np.sign(y_n) * angle_wrap_pi(np.arccos(x_n)) PK!\NdJkkpyulgresample/ulogconv.py"""Convert ulog file to different data structure.""" import pyulog import pandas as pd import re import numpy as np def create_pandadict(ULog): """Convert ulog to dictionary of topic based panda-dataframes. Rename topic-name such that each topic starts with `T_` and ends with instance ID. i.e. vehicle_local_position and instance 0 -> T_vehicle_local_position_0 rename topic-fields such that vector indicices are replaced with underline and each field starts with letter F for denoting fields i.e.: fieldmsg[0] -> F_fieldmsg_0; fieldmsg[1] -> F_fieldmsg_1 Arguments: ULog -- ulog object """ # column replacement col_rename = {"[": "_", "]": "", ".": "_"} col_rename_pattern = re.compile( r"(" + "|".join([re.escape(key) for key in col_rename.keys()]) + r")" ) pandadict = {} for msg in ULog.data_list: msg_data = pd.DataFrame.from_dict(msg.data) msg_data.columns = [ col_rename_pattern.sub(lambda x: col_rename[x.group()], col) for col in msg_data.columns ] ncol = {} for col in msg_data.columns: if col == "timestamp": ncol[col] = col else: ncol[col] = "F_" + col msg_data.rename(columns=ncol, inplace=True) msg_data.index = pd.TimedeltaIndex( msg_data["timestamp"] * 1e3, unit="ns" ) pandadict["T_{:s}_{:d}".format(msg.name, msg.multi_id)] = msg_data return pandadict def replace_nan_with_inf(ulog, topic_msgs_list): """Replace nan-values with inf-values. Arguments: pandadict -- a dictionary of pandas dataframe with keys equal to topics topic_msgs_list -- list of topicMsgs on which zero-order-hold interpolation is used """ for topic_msgs in topic_msgs_list: for ulogtopic in ulog.data_list: if ulogtopic.name == topic_msgs.topic: if topic_msgs.msgs: for msg in topic_msgs.msgs: nan_ind = np.isnan(ulogtopic.data[msg]) ulogtopic.data[msg][nan_ind] = np.inf else: for msg in ulogtopic.data.keys(): nan_ind = np.isnan(ulogtopic.data[msg]) ulogtopic.data[msg][nan_ind] = np.inf def merge_pandadict(pandadict): """Merge all dataframes within dictionanry. Arguments: pandadict -- a dictionary of pandas dataframe """ combine_topic_fieldname(pandadict) skip = True for topic in pandadict: if skip: m = pandadict[topic] skip = False else: m = pd.merge_ordered( m, pandadict[topic], on="timestamp", how="outer" ) m.index = pd.TimedeltaIndex(m.timestamp * 1e3, unit="ns") return m def apply_zoh(df, topic_msgs_list): """Apply zero-order-hold to msgs. Arguments: df -- dataframe of all msgs topic_msgs_list -- list of topicMsgs on which zoh is going to be applied """ for topicMsgs in topic_msgs_list: regex = topicMsgs.topic + ".+" if topicMsgs.msgs: regex = regex + "[" for msg in topicMsgs.msgs: regex = "{0}({1})".format(regex, msg) regex = regex + "]" df[list(df.filter(regex=regex).columns)] = df[ list(df.filter(regex=regex).columns) ].fillna(method="ffill") def combine_topic_fieldname(pandadict): """Add topic name to field-name except for timestamp field. Arguments: pandadict -- a dictionary of pandas dataframe """ for topic in pandadict.keys(): ncol = {} for col in pandadict[topic].columns: if col == "timestamp": ncol[col] = col else: ncol[col] = topic + "__" + col pandadict[topic].rename(columns=ncol, inplace=True) return PK!\Npyulgresample/ulogdataframe.py"""Create dataframe from .ulg file and convert it to other structures. Read required topics. Create ulog structure from .ulg file. Create pandas dataframe """ import os from pyulgresample import loginfo from pyulgresample import ulogconv as conv import numpy as np class TopicMsgs: """Store topic messages.""" def __init__(self, topic, msgs): """Initialization. Arguments: topic -- topic that is used to generate dataframe and ulog msgs -- list of messages from the corresponding topic """ self.topic = topic self.msgs = msgs class DfUlg: """Class that contains ulog-structure and pandas-dataframe for a set of topics. Check .ulg file. Read required topics. Create new data structures. """ def __init__(self, df, ulog, topics): """Initialization. Arguments: df -- pandas dataframe with uORB msgs from topics (resampled) ulog -- pyulog struct of uORB msgs (without resampling) topics -- list of topics that are used to generate df and ulog """ self.df = df # pandas dataframe self.ulog = ulog # ulog self.topics = topics # uorb topics @classmethod def _check_file(self, filepath): """Check if file is a .ulg file. Arguments: filepath -- path to .ulg-file """ if os.path.isfile(filepath): base, ext = os.path.splitext(filepath) if ext.lower() not in (".ulg"): raise Exception("File is not .ulg file") else: raise Exception("File does not exist") @classmethod def create( cls, filepath, topics=None, zoh_topic_msgs_list=None, nan_topic_msgs_list=None, ): """Factory method. Create a DfUlg object. By default, the merge-method uses linear interpolation for resampling. Dataframe (df) is a pandas-dataframe with index equal to the merged timestamps. Each column represents a message-field. For instance, the thrust-field of the message vehicle_local_position_setpoint message would be named as follow: T_vehicle_local_position_setpoint_0__F_thrust_x if the field x of vehicle_local_position_setpoint is a scalar or T_vehicle_local_position_setpoint_0__F_x_0 if the field x is an array, where the 0 represents the index of the array. The T stands for topic, which indicates the beginning of the topic. In this example, the topcic name is vehicle_local_position_setpoint. The topic name is followed by a number, which indicates the topic instance. If there is only one instance of a specific topic, then this number will be 0. The instance number is followed by two underlines and a capital letter F, which stands for field. In the example above, the field in question is x. Arguments: filepath -- path to .ulg file Keyword arguments: nan_topic_msgs_list -- list of TopicMsgs which contain Nan-values zoh_topic_msgs_list -- list of TopicMsgs on which zero-order-hold interpolation is used """ # check if valid file is provided cls._check_file(filepath) ulog = loginfo.get_ulog(filepath, topics) if ulog is None: raise Exception("Ulog is empty") # replace nan with inf # this is needed because inf-values are considered as numerical values and therefore are not interpolated below if nan_topic_msgs_list: conv.replace_nan_with_inf(ulog, nan_topic_msgs_list) # create pandadict pandadict = conv.create_pandadict(ulog) # merge pandadict to a complete pandaframe df = conv.merge_pandadict(pandadict) # apply zero order hold if zoh_topic_msgs_list: conv.apply_zoh(df, zoh_topic_msgs_list) # we also apply zoh for msgs, which contain nan # TODO: this is just for the time being until a better solution is found if nan_topic_msgs_list: conv.apply_zoh(df, nan_topic_msgs_list) # linearly interpolate # only NaN values get interpolated, and therefore the zero order hold values from before do not get overwritten df.interpolate(mehtod="linear", inplace=True) # after interpolation, we can replace the inf-values back to nan-values df.replace(np.inf, np.nan, inplace=True) # add seconds df["timestamp_s"] = (df.timestamp - df.timestamp[0]) * 1e-6 return cls(df, ulog, topics) PK!\N'Ɓ%pyulgresample-0.0.2.dist-info/LICENSEBSD 3-Clause License Copyright (c) 2018, YUNEEC All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PK!H>*RQ#pyulgresample-0.0.2.dist-info/WHEEL HM K-*ϳR03rOK-J,/RH,rzd&Y)r$[)T&UrPK!Hp#/&pyulgresample-0.0.2.dist-info/METADATAX[o6~ׯ 6Lɗ4Yz _P;-HkTIj߾!97t_y.߹RzQ /u;+C;҉oe"8v:aϨGIދ){7*0+|:??+%V _~z1v>JcBFXιjGZ Z ¤UCZ Y?ﲏUVV3Ԟ-z(06s95Q2@h٭~pl4bnjӶo'Y˚Dy[wWߚ +4[a C+X3딞BO~n'g'v̀bzIyE^[˞ ~C/؛'0?!?{N ] (L] &wt6?ڹVh71sI+?t+ {eVrdT_#_xHU T s\-wNbw.HH~V7z`Fx5tBnTlbM#}B5mrɳ虴~S,Ef6,a  (' Qׁ0RPDFqٝYc9QjeDXe䞘5 /1!@׬Kņ߯YY0/,uK*!FYc_/4]5tta<,pC Wv:13!WHQxԳ!xSDF+wB ZZ7iz$Nvc=ufS6vt~|aCۉ'#BY#['γ)ƅ |pf\(Ѡ*ƭcXbآs;s챹d,U5%PvѪ,Sd",3p\fZgv|:PyOKr~W2 Cs0$)DLhk,s36HD/O/*Y˜ݭ1ug"2)AZ]|ŠOvDElږb_$p+D 6EЀE<E+=2Ҁ ~K7R͒3cE(~(i@^b6";ڰ +VarQA+Q׭Y\ )ZU;h,0c`a 0FM]SQǔM Zg 65b&d.0܄*"Py D;0;-B}GER#6~C4?x[=Cͭ = 5 eeYk27KDDevΣQ.꒷܁)(PXrW 3+Qk[_( 505W"G1[sQ) !t2J2ٸ.8unA,Y_mIN18[Qo45jo@F&!F 1l7oP0XcmR㶌JW@7xU:J$S,9׆t~O LF𿠐4* FP;#_|ӒжwDј1 sv IxBwfYǃBČ8O-AݍA_9lіjS\Bꢥq㋽ x+xl.CCsx0naAvmna^ ʼnMyONצN47PSeuycK !0 ͜< ~#IypH%14͔唣Ʋ@p `DV ]T_Dz=Ǜ_xx'NI%leF<$V`Diʪ}T?y;ʼ/i]tW뱦yQ?ڍ9AtU5?m ^RXBl"r-R)ǂA9@X?ȖJJӶRiMD`N(H<a%') ?ZA!_PKKNAspyulgresample/__init__.pyPK!\N  pyulgresample/loginfo.pyPKKN]kX3 pyulgresample/mathpandas.pyPK!\NdJkk*pyulgresample/ulogconv.pyPK!\N9pyulgresample/ulogdataframe.pyPK!\N'Ɓ%Kpyulgresample-0.0.2.dist-info/LICENSEPK!H>*RQ#Rpyulgresample-0.0.2.dist-info/WHEELPK!Hp#/&Rpyulgresample-0.0.2.dist-info/METADATAPK!HFTM$[pyulgresample-0.0.2.dist-info/RECORDPK  ]