Compare 1b7032b ... +37 ... 8d46c9c

            section_buses = list(net.bus[net.bus.index.isin(section)

                                         & (net.bus.in_service == True)].index)

            section_gens = list(net.gen[net.gen.bus.isin(section)

                                        & (net.gen.in_service == True)].index)

            section_sgens = list(net.sgen[net.sgen.bus.isin(section)

                                          & (net.sgen.in_service == True)].index)

            section_loads = list(net.load[net.load.bus.isin(section)

                                          & (net.load.in_service == True)].index)

    isolated_trafos_is = isolated_trafos.intersection((set(net.trafo[net.trafo.in_service == True]

                                                           .index)))

    isolated_trafos3w = set(

        (open_trafo_switches.groupby("element").count().query("bus > 2").index))

    isolated_trafos3w_is = isolated_trafos3w.intersection((

        set(net.trafo[net.trafo.in_service == True].index)))

def print_progress_bar(iteration, total, prefix='', suffix='', decimals=1, length=100, fill='█'):

    """

    Call in a loop to create terminal progress bar.

    the code is mentioned in : https://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console

    """

    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))

    filled_length = int(length * iteration // total)

    bar = fill * filled_length + '-' * (length - filled_length)

    # logger.info('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix))

    print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end="")

    # Print New Line on Complete

    if iteration == total:

        print("\n")

    eg_oos = net.ext_grid[net.ext_grid.in_service == False].index

    open_switches = (net.switch.closed.values == False)

        switch_mask = open_switches & (net.switch.et.values == et)

from pandapower.run import rundcpp, runpp, runopp

from pandapower.powerflow import LoadflowNotConverged

from pandapower.optimal_powerflow import OPFNotConverged

from pandapower.toolbox import get_element_index

try:

    from pypower import ppoption, runpf, runopf, rundcpf, rundcopf

    ppopt = ppoption.ppoption(VERBOSE=0, OUT_ALL=0)

    pypower_import = True

except ImportError:

    pypower_import = False

    INPUT:

        **ppc** : The pypower case file.

    OPTIONAL:

        **f_hz** (float, 50) - The frequency of the network.

        **validate_conversion** (bool, False) - If True, validate_from_ppc is run after conversion.

            For running the validation, the ppc must already contain the pypower

            powerflow results or pypower must be installed.

        ****kwargs** keyword arguments for

    OUTPUT:

        **net** : pandapower net.

    EXAMPLE:

        import pandapower.converter as pc

        from pypower import case4gs

        ppc_net = case4gs.case4gs()

        net = pc.from_ppc(ppc_net, f_hz=60)

    gen_lookup = _from_ppc_gen(net, ppc)

    _from_ppc_branch(net, ppc, f_hz, **kwargs)

    _from_ppc_gencost(net, ppc, gen_lookup, check=kwargs.get("check_costs", True))

    net._options = {}

    net._options["gen_lookup"] = gen_lookup

        net, ppc['bus'].shape[0], name=ppc['bus'][:, BUS_I].astype(int),

        max_vm_pu=ppc['bus'][:, VMAX], min_vm_pu=ppc['bus'][:, VMIN])

    # unused data of ppc: Vm, Va (partwise: in ext_grid), zone

            net, bus=bus_pos[i], vm_pu=vg_bus_lookup.at[bus_pos[i]],

            max_q_mvar=ppc['gen'][i, QMAX], min_q_mvar=ppc['gen'][i, QMIN]))

        net, buses=bus_pos[is_gen], vm_pu=vg_bus_lookup.loc[bus_pos[is_gen]].values,

        max_q_mvar=ppc['gen'][is_gen, QMAX], min_q_mvar=ppc['gen'][is_gen, QMIN])

        net, buses=bus_pos[is_sgen], p_mw=ppc['gen'][is_sgen, PG],

        controllable=True)

    # unused data of ppc: Vg (partwise: in ext_grid and gen), mBase, Pc1, Pc2, Qc1min, Qc1max,

    gen_lookup = pd.DataFrame({

        'element': np.r_[idx_eg, idx_gen, idx_sgen],

        'element_type': ["ext_grid"]*sum(is_ext_grid) + ["gen"]*sum(is_gen) + ["sgen"]*sum(is_sgen)

        })

    create_lines_from_parameters(

        net, from_buses=from_bus[is_line], to_buses=to_bus[is_line], length_km=1,

        create_transformers_from_parameters(

            net, hv_buses=hv_bus, lv_buses=lv_bus, sn_mva=sn,

            vn_hv_kv=vn_hv_kv, vn_lv_kv=vn_lv_kv,

    # unused data of ppc: rateB, rateC

    if not 'gencost' in ppc:

    to_iterate = set(max_diff_values.keys()) & {'gen_q_mvar', 'branch_p_mw', 'branch_q_mvar',

                                                'gen_p_mw', 'bus_va_degree', 'bus_vm_pu'}

    if not len(to_iterate):

        logger.warning("There are no keys to validate.")

    for i in to_iterate:

        elm = i.split("_")[0]

        sought = ["p", "q"] if elm != "bus" else ["vm", "va"]

        col = int(np.array([0, 1])[[j in i for j in sought]][0]) if elm != "branch" else \

            list(np.array([[0, 2], [1, 3]])[[j in i for j in sought]][0])

        val &= bool(np.max(abs(diff_res[elm][:, col])) < max_diff_values[i])

def _gen_bus_info(ppc, idx_gen):

    bus_name = int(ppc["gen"][idx_gen, GEN_BUS])

    # assumption: there is only one bus with this bus_name:

    idx_bus = int(np.where(ppc["bus"][:, BUS_I] == bus_name)[0][0])

    current_bus_type = int(ppc["bus"][idx_bus, 1])

    same_bus_gen = np.where(ppc["gen"][:, GEN_BUS] == ppc["gen"][idx_gen, GEN_BUS])[0].astype(int)

    same_bus_gen = same_bus_gen[np.where(ppc["gen"][same_bus_gen, GEN_STATUS] > 0)]

    first_same_bus = same_bus_gen[0] if len(same_bus_gen) else None

    return current_bus_type, idx_bus, first_same_bus

def validate_from_ppc(ppc_net, net, pf_type="runpp", max_diff_values={

    "bus_vm_pu": 1e-6, "bus_va_degree": 1e-5, "branch_p_mw": 1e-6, "branch_q_mvar": 1e-6,

        "gen_p_mw": 1e-6, "gen_q_mvar": 1e-6}, run=True):

    This function validates the pypower case files to pandapower net structure conversion via a \

    comparison of loadflow calculation results. (Hence the opf cost conversion is not validated.)

    INPUT:

        **ppc_net** - The pypower case file, which must already contain the pypower powerflow

            results or pypower must be importable.

        **net** - The pandapower network.

    OPTIONAL:

        **pf_type** ("runpp", string) - Type of validated power flow. Possible are ("runpp",

            "rundcpp", "runopp", "rundcopp")

        **max_diff_values** - Dict of maximal allowed difference values. The keys must be

        'vm_pu', 'va_degree', 'p_branch_mw', 'q_branch_mvar', 'p_gen_mw' and 'q_gen_mvar' and

        the values floats.

        **run** (True, bool or list of two bools) - changing the value to False avoids trying to run

            (optimal) loadflows. Giving a list of two bools addresses first pypower and second

            pandapower.

    OUTPUT:

        **conversion_success** - conversion_success is returned as False if pypower or pandapower

        cannot calculate a powerflow or if the maximum difference values (max_diff_values )

        cannot be hold.

    EXAMPLE:

        import pandapower.converter as pc

        net = cv.from_ppc(ppc_net, f_hz=50)

        conversion_success = cv.validate_from_ppc(ppc_net, net)

    NOTE:

        The user has to take care that the loadflow results already are included in the provided \

        ppc_net or pypower is importable.

    # check in case of optimal powerflow comparison whether cost information exist

    if "opp" in pf_type:

        if not (len(net.polynomial_cost) | len(net.piecewise_linear_cost)):

            if "gencost" in ppc_net:

                if not len(ppc_net["gencost"]):

                    logger.debug('ppc and pandapower net do not include cost information.')

                    return True

                else:

                    logger.error('The pandapower net does not include cost information.')

                    return False

            else:

                logger.debug('ppc and pandapower net do not include cost information.')

                return True

    # guarantee run parameter as list, for pypower and pandapower (optimal) powerflow run

    run = [run, run] if isinstance(run, bool) else run

    # --- check pypower powerflow success, if possible

    if pypower_import and run[0]:

        try:

            if pf_type == "runpp":

                ppc_net = runpf.runpf(ppc_net, ppopt)[0]

            elif pf_type == "rundcpp":

                ppc_net = rundcpf.rundcpf(ppc_net, ppopt)[0]

            elif pf_type == "runopp":

                ppc_net = runopf.runopf(ppc_net, ppopt)

            elif pf_type == "rundcopp":

                ppc_net = rundcopf.rundcopf(ppc_net, ppopt)

            else:

                raise ValueError(f"The pf_type {pf_type} is unknown")

        except:

            logger.debug("The pypower run did not work.")

    ppc_success = True

    if 'success' in ppc_net.keys():

        if ppc_net['success'] != 1:

            ppc_success = False

            logger.error("The given ppc data indicates an unsuccessful pypower powerflow: " +

                         "'ppc_net['success'] != 1'")

    if (ppc_net['branch'].shape[1] < 17):

        ppc_success = False

        logger.error("The shape of given ppc data indicates missing pypower powerflow results.")

    # --- try to run a pandapower powerflow

    if run[1]:

        if pf_type == "runpp":

            try:

                runpp(net, init="dc", calculate_voltage_angles=True, trafo_model="pi")

            except LoadflowNotConverged:

                try:

                    runpp(net, calculate_voltage_angles=True, init="flat", trafo_model="pi")

                except LoadflowNotConverged:

                    try:

                        runpp(net, trafo_model="pi", calculate_voltage_angles=False)

                        if "bus_va_degree" in max_diff_values.keys():

                            max_diff_values["bus_va_degree"] = 1e2 if max_diff_values[

                                "bus_va_degree"] < 1e2 else max_diff_values["bus_va_degree"]

                        logger.info("voltage_angles could be calculated.")

                    except LoadflowNotConverged:

                        logger.error('The pandapower powerflow does not converge.')

        elif pf_type == "rundcpp":

            try:

                rundcpp(net, trafo_model="pi")

            except LoadflowNotConverged:

                logger.error('The pandapower dc powerflow does not converge.')

        elif pf_type == "runopp":

                try:

                    runopp(net, init="flat", calculate_voltage_angles=True)

                except OPFNotConverged:

                    try:

                        runopp(net, init="pf", calculate_voltage_angles=True)

                    except (OPFNotConverged, LoadflowNotConverged, KeyError):

                        try:

                            runopp(net, init="flat", calculate_voltage_angles=False)

                            logger.info("voltage_angles could be calculated.")

                            if "bus_va_degree" in max_diff_values.keys():

                                max_diff_values["bus_va_degree"] = 1e2 if max_diff_values[

                                    "bus_va_degree"] < 1e2 else max_diff_values["bus_va_degree"]

                        except OPFNotConverged:

                            try:

                                runopp(net, init="pf", calculate_voltage_angles=False)

                                if "bus_va_degree" in max_diff_values.keys():

                                    max_diff_values["bus_va_degree"] = 1e2 if max_diff_values[

                                        "bus_va_degree"] < 1e2 else max_diff_values["bus_va_degree"]

                                logger.info("voltage_angles could be calculated.")

                            except (OPFNotConverged, LoadflowNotConverged, KeyError):

                                logger.error('The pandapower optimal powerflow does not converge.')

        elif pf_type == "rundcopp":

            try:

                rundcopp(net)

            except LoadflowNotConverged:

                logger.error('The pandapower dc optimal powerflow does not converge.')

        else:

            raise ValueError("The pf_type %s is unknown" % pf_type)

    # --- prepare powerflow result comparison by reordering pp results as they are in ppc results

    if not ppc_success:

        return False

    if "opp" in pf_type:

        if not net.OPF_converged:

            return

    elif not net.converged:

        return False

    # --- store pypower powerflow results

    ppc_res["branch"] = ppc_net['branch'][:, 13:17]

    ppc_res["bus"] = ppc_net['bus'][:, 7:9]

    ppc_res["gen"] = ppc_net['gen'][:, 1:3]

    # --- pandapower bus result table

    pp_res["bus"] = np.array(net.res_bus.sort_index()[['vm_pu', 'va_degree']])

    # --- pandapower gen result table

    pp_res["gen"] = np.zeros([1, 2])

    # consideration of parallel generators via storing how much generators have been considered

    # each node

    # if in ppc is only one gen -> numpy initially uses one dim array -> change to two dim array

    if len(ppc_net["gen"].shape) == 1:

        ppc_net["gen"] = np.array(ppc_net["gen"], ndmin=2)

    GENS = pd.DataFrame(ppc_net['gen'][:, [0]].astype(int))

    GEN_uniq = GENS.drop_duplicates()

    already_used_gen = pd.Series(np.zeros(GEN_uniq.shape[0]).astype(int),

                                 index=[int(v) for v in GEN_uniq.values])

    change_q_compare = []

    for i, j in GENS.iterrows():

        current_bus_type, current_bus_idx, first_same_bus_in_service_gen_idx, = _gen_bus_info(

            ppc_net, i)

        if current_bus_type == 3 and i == first_same_bus_in_service_gen_idx:

            pp_res["gen"] = np.append(pp_res["gen"], np.array(net.res_ext_grid[

                    net.ext_grid.bus == current_bus_idx][['p_mw', 'q_mvar']]).reshape((1, 2)), 0)

        elif current_bus_type == 2 and i == first_same_bus_in_service_gen_idx:

            pp_res["gen"] = np.append(pp_res["gen"], np.array(net.res_gen[

                    net.gen.bus == current_bus_idx][['p_mw', 'q_mvar']]).reshape((1, 2)), 0)

            pp_res["gen"] = np.append(pp_res["gen"], np.array(net.res_sgen[

                net.sgen.bus == current_bus_idx][['p_mw', 'q_mvar']])[

                already_used_gen.at[int(j)]].reshape((1, 2)), 0)

            already_used_gen.at[int(j)] += 1

            change_q_compare += [int(j)]

    pp_res["gen"] = pp_res["gen"][1:, :]  # delete initial zero row

    # --- pandapower branch result table

    pp_res["branch"] = np.zeros([1, 4])

    # consideration of parallel branches via storing how often branches were considered

    # each node-to-node-connection

    try:

        init1 = pd.concat([net.line.from_bus, net.line.to_bus], axis=1,

                          sort=True).drop_duplicates()

        init2 = pd.concat([net.trafo.hv_bus, net.trafo.lv_bus], axis=1,

                          sort=True).drop_duplicates()

    except TypeError:

        # legacy pandas < 0.21

        init1 = pd.concat([net.line.from_bus, net.line.to_bus], axis=1).drop_duplicates()

        init2 = pd.concat([net.trafo.hv_bus, net.trafo.lv_bus], axis=1).drop_duplicates()

    init1['hv_bus'] = np.nan

    init1['lv_bus'] = np.nan

    init2['from_bus'] = np.nan

    init2['to_bus'] = np.nan

    try:

        already_used_branches = pd.concat([init1, init2], axis=0, sort=True)

    except TypeError:

        # pandas < 0.21 legacy

        already_used_branches = pd.concat([init1, init2], axis=0)

    already_used_branches['number'] = np.zeros([already_used_branches.shape[0], 1]).astype(int)

    BRANCHES = pd.DataFrame(ppc_net['branch'][:, [0, 1, TAP, SHIFT]])

    for i in BRANCHES.index:

        from_bus = get_element_index(net, 'bus', name=int(ppc_net['branch'][i, 0]))

        to_bus = get_element_index(net, 'bus', name=int(ppc_net['branch'][i, 1]))

        from_vn_kv = ppc_net['bus'][from_bus, BASE_KV]

        to_vn_kv = ppc_net['bus'][to_bus, BASE_KV]

        ratio = BRANCHES[2].at[i]

        angle = BRANCHES[3].at[i]

        # from line results

        if (from_vn_kv == to_vn_kv) & ((ratio == 0) | (ratio == 1)) & (angle == 0):

            pp_res["branch"] = np.append(pp_res["branch"], np.array(net.res_line[

                (net.line.from_bus == from_bus) &

                (net.line.to_bus == to_bus)]

                [['p_from_mw', 'q_from_mvar', 'p_to_mw', 'q_to_mvar']])[

                int(already_used_branches.number.loc[

                   (already_used_branches.from_bus == from_bus) &

                   (already_used_branches.to_bus == to_bus)].values)].reshape(1, 4), 0)

            already_used_branches.number.loc[(already_used_branches.from_bus == from_bus) &

                                             (already_used_branches.to_bus == to_bus)] += 1

        # from trafo results

        else:

            if from_vn_kv >= to_vn_kv:

                pp_res["branch"] = np.append(pp_res["branch"], np.array(net.res_trafo[

                    (net.trafo.hv_bus == from_bus) &

                    (net.trafo.lv_bus == to_bus)]

                    [['p_hv_mw', 'q_hv_mvar', 'p_lv_mw', 'q_lv_mvar']])[

                    int(already_used_branches.number.loc[

                        (already_used_branches.hv_bus == from_bus) &

                        (already_used_branches.lv_bus == to_bus)].values)].reshape(1, 4), 0)

                already_used_branches.number.loc[(already_used_branches.hv_bus == from_bus) &

                                                 (already_used_branches.lv_bus == to_bus)] += 1

            else:  # switch hv-lv-connection of pypower connection buses

                pp_res["branch"] = np.append(pp_res["branch"], np.array(net.res_trafo[

                    (net.trafo.hv_bus == to_bus) &

                    (net.trafo.lv_bus == from_bus)]

                    [['p_lv_mw', 'q_lv_mvar', 'p_hv_mw', 'q_hv_mvar']])[

                    int(already_used_branches.number.loc[

                        (already_used_branches.hv_bus == to_bus) &

                        (already_used_branches.lv_bus == from_bus)].values)].reshape(1, 4), 0)

                already_used_branches.number.loc[

                    (already_used_branches.hv_bus == to_bus) &

                    (already_used_branches.lv_bus == from_bus)] += 1

    pp_res["branch"] = pp_res["branch"][1:, :]  # delete initial zero row

    # --- do the powerflow result comparison

    diff_res = dict.fromkeys(ppc_elms)

    diff_res["bus"] = ppc_res["bus"] - pp_res["bus"]

    diff_res["bus"][:, 1] -= diff_res["bus"][0, 1]  # remove va_degree offset

    diff_res["branch"] = ppc_res["branch"] - pp_res["branch"]

    diff_res["gen"] = ppc_res["gen"] - pp_res["gen"]

    # comparison of buses with several generator units only as q sum

    for i in GEN_uniq.loc[GEN_uniq[0].isin(change_q_compare)].index:

        next_is = GEN_uniq.index[GEN_uniq.index > i]

        if len(next_is) > 0:

            next_i = next_is[0]

        else:

            next_i = GENS.index[-1] + 1

        if (next_i - i) > 1:

            diff_res["gen"][i:next_i, 1] = sum(diff_res["gen"][i:next_i, 1])

    # logger info

    logger.debug("Maximum voltage magnitude difference between pypower and pandapower: "

                 "%.2e pu" % np.max(abs(diff_res["bus"][:, 0])))

    logger.debug("Maximum voltage angle difference between pypower and pandapower: "

                 "%.2e degree" % np.max(abs(diff_res["bus"][:, 1])))

    logger.debug("Maximum branch flow active power difference between pypower and pandapower: "

                 "%.2e MW" % np.max(abs(diff_res["branch"][:, [0, 2]])))

    logger.debug("Maximum branch flow reactive power difference between pypower and "

                 "pandapower: %.2e MVAr" % np.max(abs(diff_res["branch"][:, [1, 3]])))

    logger.debug("Maximum active power generation difference between pypower and pandapower: "

                 "%.2e MW" % np.max(abs(diff_res["gen"][:, 0])))

    logger.debug("Maximum reactive power generation difference between pypower and pandapower: "

                 "%.2e MVAr" % np.max(abs(diff_res["gen"][:, 1])))

    if _validate_diff_res(diff_res, {"bus_vm_pu": 1e-3, "bus_va_degree": 1e-3, "branch_p_mw": 1e-6,

                                     "branch_q_mvar": 1e-6}) and \

            (np.max(abs(diff_res["gen"])) > 1e-1).any():

        logger.debug("The active/reactive power generation difference possibly results "

                     "because of a pypower error. Please validate "

                     "the results via pypower loadflow.")  # this occurs e.g. at ppc case9

    # give a return

    if isinstance(max_diff_values, dict):

        return _validate_diff_res(diff_res, max_diff_values)

    else:

        logger.debug("'max_diff_values' must be a dict.")