Import data from a text file into a pandas dataframe

manpreet Best Answer 2 years ago

I'm building a web app using Django. I uploaded a text file using

csv_file = request.FILES['file'].

I can't read the csv into pandas. The file that i'm trying to import has text and data, but I only want the data.

I've tried the following

1. df = pd.read_csv(csv_file, sep=" ", header=None, names=["col1","col2","col3"], skiprows = 2) to try to remove the comments and just read the numbers

Error: pandas will not read all 3 columns. It only reads 1 column

2. I tried df = pd.read_csv(csv_file, sep="\s{2}", sep=" ", header=None, names=["col1","col2","col3"], skiprows = 2) to try to remove the comments and just read the numbers

Error: cannot use a string pattern on a bytes-like object

3. I tried df = pd.read_csv(csv_file.read(), sep=" ", header=None, names=["col1","col2","col3"], skiprows = 2) to try to remove the comments and just read the numbers

File I uploaded

% filename
% username
2.0000  117.441  -0.430
2.0100  117.499  -0.337
2.0200  117.557  -0.246
2.0300  117.615  -0.157
2.0400  117.672  -0.069

views.py

def new_measurement(request, pk):
    material = Material.objects.get(pk=pk)
    if request.method == 'POST':
        form = NewTopicForm(request.POST)
        if form.is_valid():
            topic = form.save(commit=False)
            topic.material = material
            topic.message=form.cleaned_data.get('message')
            csv_file = request.FILES['file']
            df = genDataFrame(csv_file)
            topic.data = df
            topic.created_by = request.user
            topic.save()
            return redirect('topic_detail', pk =  material.pk)
    else:
        form = NewTopicForm()
    return render(request, 'new_topic.html', {'material': material, 'form': form})

def genDataFrame(csv_file):
    df = pd.read_csv(csv_file, sep=" ", header=None, names=["col1","col2","col3"])
    df = df.convert_objects(convert_numeric=True)
    df = df.dropna()
    df = df.reset_index(drop = True)
    return df_list