Comment diviser un bloc de données en groupes de taille fixe?

Cela vous donnera une liste de DataFrames:

lst = [df.iloc[i:i+group_size] for i in range(0,len(df),group_size)]
if len(lst[-1]) < group_size:
   lst.pop()

Il utilise juste l'indexation intégrée, donc cela devrait être assez rapide. L'agitation avec l'index d'arrêt prend soin de supprimer la dernière image si elle est trop petite - vous pouvez également la décomposer avec

lst = [df.iloc[i:i+group_size] for i in range(0,len(df)-group_size+1,group_size)]

Délimitez avec une tranche, puis ffill ().

import pandas as pd
from datetime import datetime as dt
print(pd.__version__)


def test1():
    df = pd.DataFrame({'a':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]})  # data frame has been sorted before this point and the rows are in the correct order
    #print(df)
    group_size = 3
    numbers = list(range(len(df.index) // group_size)) * group_size
    numbers.sort()
    numbers = pd.Series(numbers)
    df = pd.concat([df, numbers], ignore_index=True, axis=1)
    df.columns = ['value', 'group number']
    groups = df.groupby('group number').filter(lambda x: len(x) == group_size)
    #print(groups)

def test2():
    # Won't work well because there is no easy way to calculate the remainder that should
    # not be grouped.  But cut() is good for discretizing continuous values
    df = pd.DataFrame({'a':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]})  # data frame has been sorted before this point and the rows are in the correct order
    num_groups = len(df.index)/3
    df['group'] = pd.cut(df['a'], num_groups, right=False)
    #print(df)

def test3():
    # df has a RangeIndex, so we get to slice 
    df = pd.DataFrame({'a':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]})  # data frame has been sorted before this point and the rows are in the correct order
    df['group'] = df[::3]
    df['group'].ffill(inplace=True)
    #print(df['group'])

def test4():
    # A mask can also be used
    df = pd.DataFrame({'a':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]})  # data frame has been sorted before this point and the rows are in the correct order
    df['group'] = df[df.index % 3 == 0]
    df['group'].ffill(inplace=True)
    #print(df)

def test5():
    # maybe go after grouping with iloc
    df = pd.DataFrame({'a':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]})  # data frame has been sorted before this point and the rows are in the correct order
    group = 0
    for i in range(0,len(df), 3):
        df.loc[i:i+3, 'group'] = group
        group+=1
    #print(df)


funcs = [test1, test2, test3, test4, test5]
for func in funcs:
    print(func.__name__)
    a = dt.now()
    for i in range(1000):
        func()
    b = dt.now()
    print(b-a)

Vous pouvez maintenant faire un regroupement et supprimer les groupes qui sont trop petits.

# df has a RangeIndex, so we get to slice 
group_size = 3
df = pd.DataFrame({'a':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]})  # data frame has been sorted before this point and the rows are in the correct order
slices = df[::group_size]

# but you don't want the group number to be the ordinal at the slices
# so make a copy of the slice to assign good group numbers to it (or get a chained assignment warning)
slices=slices.copy()
slices['group'] = [i for i in range(len(slices))]
df['group'] = slices['group']

# ffill with the nice group numbers
df['group'].ffill(inplace=True)

#now trim the last group
last_group = df['group'].max()
if len(df[df['group']==last_group]) < group_size:
    df = df[df['group'] != last_group]

print(df)

Ceci est une variante de la réponse de Perigon. Dans mon cas, je ne voulais pas jeter les derniers, donc cela montre comment mettre le reste dans une liste finale. Je lisais un CSV et je voulais faire du multitraitement, je vais donc passer les plus petites trames de données à des processus séparés et je ne peux pas perdre de lignes du CSV. Donc, dans mon cas, le nombre de processus désiré par groupe est défini sur le même nombre de processus que je souhaite traiter.

    import pandas as pd
    
    test_dict = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]
    
    
    df = pd.DataFrame.from_dict(test_dict)
    
    print ('Size of dataFrame=', len(df.index))
    desired_number_of_groups = 4
    group_size = int(len(df.index) / (desired_number_of_groups))
    print("group_size=", group_size)
    remainder_size = len(df.index) % group_size
    print("remainder_size=", remainder_size)
    df_split_list = [df.iloc[i:i + group_size] for i in range(0, len(df) - group_size + 1, group_size)]
    print("Number of split_dataframes=", len(df_split_list))
    if remainder_size > 0:
        df_remainder = df.iloc[-remainder_size:len(df.index)]
        df_split_list.append(df_remainder)
    print("Revised Number of split_dataframes=", len(df_split_list))
    print("Splitting complete, verifying counts")
    
    count_all_rows_after_split = 0
    for index, split_df in enumerate(df_split_list):
        print("split_df:", index, " size=", len(split_df.index))
        count_all_rows_after_split += len(split_df.index)
    
    if count_all_rows_after_split != len(df.index):
        raise Exception('count_all_rows_after_split = ', count_all_rows_after_split,
                         " but original CSV DataFrame has count =", len(df.index)
                         )

Rich a fait un meilleur travail avec ses cas de test unitaires. Je viens de tester test_dict avec 1:17, puis 1:18, puis 1:19, puis 1:20, puis 1:21)