Auto-detect tab-delimited CSV files with manual delimiter override

packages/sanddance-explorer/src/dataLoader.ts

@@ -11,20 +11,32 @@ import {
 } from './interfaces';
 import { SandDance } from '@msrvida/sanddance-react';
 
+export function guessDelimiterType(text: string, type: DataFileType): DataFileType {
+    if (type !== 'csv') return type;
+    const firstLine = text.split(/\r?\n/)[0];
+    // Strip quoted strings to avoid counting delimiters inside quotes
+    const unquoted = firstLine.replace(/"[^"]*"/g, '""');
+    const tabCount = (unquoted.match(/\t/g) || []).length;
+    const commaCount = (unquoted.match(/,/g) || []).length;
+    return tabCount > commaCount ? 'tsv' : 'csv';
+}
+
 export const loadDataFile = (dataFile: DataFile, columnTypes?: SandDance.types.ColumnTypeMap) => new Promise<DataContent>((resolve, reject) => {
     const vega = SandDance.VegaMorphCharts.base.vega;
     const loader = vega.loader();
 
     function handleRawText(text: string) {
         let data: object[];
+        const type = dataFile.noTypeGuess ? dataFile.type : guessDelimiterType(text, dataFile.type);
         try {
-            data = vega.read(text, { type: dataFile.type, parse: {} });
+            data = vega.read(text, { type, parse: {} });
         }
         catch (e) {
             reject(e);
         }
         if (data) {
-            loadDataArray(data, dataFile.type, columnTypes).then(dc => {
+            loadDataArray(data, type, columnTypes).then(dc => {
+                dc.type = type; // communicate the actual parsed type back to the caller
                 if (dataFile.snapshotsUrl) {
                     fetch(dataFile.snapshotsUrl)
                         .then(response => response.json())

packages/sanddance-explorer/src/dialogs/dataBrowser.tsx

@@ -4,7 +4,7 @@
 */
 
 import { base } from '../base';
-import { DataExportHandler } from '../interfaces';
+import { DataExportHandler, DataFile, DataFileType } from '../interfaces';
 import { DataExportPicker, removeExtensions } from '../controls/dataExporter';
 import { DataItem } from '../controls/dataItem';
 import { DataScopeId } from '../controls/dataScope';
@@ -49,6 +49,8 @@ export interface Props {
     displayName: string;
     explorer: Explorer_Class;
     onUpdateColumnTypes: (columnTypes: SandDance.types.ColumnTypeMap) => void;
+    dataFile?: DataFile;
+    onReloadFileType?: (type: DataFileType) => void;
 }
 
 export function DataBrowser(props: Props) {
@@ -87,6 +89,27 @@ export function DataBrowser(props: Props) {
                     props.onDataScopeClick(o.key as DataScopeId);
                 }}
             />
+            {props.dataFile && (props.dataFile.type === 'csv' || props.dataFile.type === 'tsv') && props.onReloadFileType && (
+                <Dropdown
+                    label={strings.labelDelimiter}
+                    collapseLabel={true}
+                    options={[
+                        {
+                            key: 'csv',
+                            text: strings.labelDelimiterComma,
+                            isSelected: props.dataFile.type === 'csv',
+                        },
+                        {
+                            key: 'tsv',
+                            text: strings.labelDelimiterTab,
+                            isSelected: props.dataFile.type === 'tsv',
+                        },
+                    ]}
+                    onChange={(e, o) => {
+                        props.onReloadFileType(o.key as DataFileType);
+                    }}
+                />
+            )}
             {!props.data && <div dangerouslySetInnerHTML={{ __html: props.nullMessage }}></div>}
             {props.data && !props.data.length && <div>{props.zeroMessage}</div>}
             {!!length && <div>

packages/sanddance-explorer/src/explorer.tsx

@@ -495,7 +495,7 @@ function _Explorer(_props: Props) {
                     const newState: Partial<State> = {
                         camera: undefined,
                         columns: {},
-                        dataFile,
+                        dataFile: dataContent.type ? { ...dataFile, type: dataContent.type } : dataFile,
                         dataContent,
                         snapshots: dataContent.snapshots || this.state.snapshots,
                         autoCompleteDistinctValues: {},
@@ -1284,6 +1284,11 @@ function _Explorer(_props: Props) {
                                                 onUpdateColumnTypes={columnTypes => {
                                                     this.load(this.state.dataFile, null, { prefs: this.prefs, columnTypes });
                                                 }}
+                                                dataFile={this.state.dataFile}
+                                                onReloadFileType={type => {
+                                                    const dataFile = { ...this.state.dataFile, type, noTypeGuess: true };
+                                                    this.load(dataFile, null, { prefs: this.prefs });
+                                                }}
                                             />
                                         );
                                     }

packages/sanddance-explorer/src/interfaces.ts

@@ -19,12 +19,14 @@ export interface DataFile {
   rawText?: string;
   snapshots?: Snapshot[];
   type: DataFileType;
+  noTypeGuess?: boolean;
 }
 
 export interface DataContent {
   data: object[];
   columns: SandDance.types.Column[];
   snapshots?: Snapshot[];
+  type?: DataFileType;
 }
 
 export type DataExportType = DataFileType | 'html';

packages/sanddance-explorer/src/language.ts

@@ -90,6 +90,9 @@ export const strings = {
     labelColumnQuantitativeMean: 'Mean',
     labelDataBrowser: 'Data browser',
     labelDataScope: 'Scope',
+    labelDelimiter: 'Delimiter',
+    labelDelimiterComma: 'Comma (,)',
+    labelDelimiterTab: 'Tab',
     labelExport: 'Export Data',
     labelExportFormat: 'File format',
     labelExportCSV: '.CSV - Comma separated values',