001/*
002 * Stallion Core: A Modern Web Framework
003 *
004 * Copyright (C) 2015 - 2016 Stallion Software LLC.
005 *
006 * This program is free software: you can redistribute it and/or modify it under the terms of the
007 * GNU General Public License as published by the Free Software Foundation, either version 2 of
008 * the License, or (at your option) any later version. This program is distributed in the hope that
009 * it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
010 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
011 * License for more details. You should have received a copy of the GNU General Public License
012 * along with this program.  If not, see <http://www.gnu.org/licenses/gpl-2.0.html>.
013 *
014 *
015 *
016 */
017
018package io.stallion.dataAccess.file;
019
020import com.moandjiezana.toml.Toml;
021import com.moandjiezana.toml.TomlWriter;
022import io.stallion.Context;
023import io.stallion.dataAccess.ModelController;
024import io.stallion.services.Log;
025import io.stallion.services.PermaCache;
026import io.stallion.settings.Settings;
027import io.stallion.utils.Literals;
028import io.stallion.utils.Markdown;
029import io.stallion.utils.GeneralUtils;
030import io.stallion.utils.json.JSON;
031import org.apache.commons.codec.digest.DigestUtils;
032import org.apache.commons.io.FilenameUtils;
033import org.apache.commons.lang3.StringUtils;
034
035import java.io.BufferedReader;
036import java.io.File;
037import java.io.IOException;
038import java.nio.charset.StandardCharsets;
039import java.nio.file.FileSystems;
040import java.nio.file.Files;
041import java.nio.file.Path;
042import java.nio.file.Paths;
043import java.time.LocalDateTime;
044import java.time.ZoneId;
045import java.time.ZonedDateTime;
046import java.time.format.DateTimeFormatter;
047import java.time.format.DateTimeParseException;
048import java.util.*;
049import java.util.regex.Matcher;
050import java.util.regex.Pattern;
051
052
053import static io.stallion.utils.Literals.*;
054
055
056
057public class TextFilePersister<T extends TextItem> extends FilePersisterBase<T> {
058    private String bucket;
059    private Class<T> clazz = null;
060    private ModelController controller;
061    private String targetPath;
062
063    private static DateTimeFormatter[] localDateFormats = new DateTimeFormatter[] {
064            DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"),
065            DateTimeFormatter.ofPattern("yyyy-MM-dd"),
066            DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
067    };
068    private static DateTimeFormatter[] zonedDateFormats = new DateTimeFormatter[] {
069            DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm z"),
070            DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm VV"),
071            DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z")
072    };
073
074
075    private static final Set<String> allowedExtensions = set("md", "txt", "html", "htm");
076
077
078    @Override
079    public Set<String> getFileExtensions() {
080        return allowedExtensions;
081    }
082
083    @Override
084    public T doFetchOne(File file) {
085        Path path = FileSystems.getDefault().getPath(file.getAbsolutePath());
086        BufferedReader reader = null;
087        try {
088            reader = Files.newBufferedReader(path, StandardCharsets.UTF_8);
089        } catch (IOException e) {
090            throw new RuntimeException(e);
091        }
092        int lineCount = 0;
093        StringBuffer buffer = new StringBuffer();
094        for(int i: safeLoop(50000)) {
095            String line = null;
096            try {
097                line = reader.readLine();
098            } catch (IOException e) {
099                throw new RuntimeException(e);
100            }
101            if (line == null) {
102                break;
103            }
104            buffer.append(line + "\n");
105            lineCount = i;
106        }
107        if (lineCount < 2) {
108            return null;
109        }
110        String fileContent = buffer.toString();
111        return fromString(fileContent, path);
112    }
113
114
115
116    @Override
117    public void persist(T obj) {
118        if (obj.getId() == null) {
119            obj.setId(Context.dal().getTickets().nextId());
120        }
121        String filePath = fullFilePathForObj(obj);
122        TextItem item = (TextItem)obj;
123        String s = null;
124        if (filePath.endsWith(".html") || filePath.endsWith(".html")) {
125            s = toHtml(item);
126        } else {
127            s = stringify(item);
128        }
129        try {
130            Files.write(Paths.get(filePath), s.getBytes());
131        } catch (IOException e) {
132            throw new RuntimeException(e);
133        }
134    }
135
136
137
138    private static Pattern tomlPattern = Pattern.compile("\\-\\-\\-toml\\-\\-\\-([\\s\\S]*)\\-\\-\\-end\\-toml\\-\\-\\-");
139    public T fromString(String fileContent, Path fullPath) {
140        if (fullPath.toString().endsWith(".html") || fullPath.toString().endsWith(".htm")) {
141            return fromHtml(fileContent, fullPath);
142        }
143        String relativePath = fullPath.toString().replace(getBucketFolderPath(), "");
144        Path path = fullPath;
145
146        T item = null;
147        try {
148            item = getModelClass().newInstance();
149        } catch (InstantiationException e) {
150            throw new RuntimeException(e);
151        } catch (IllegalAccessException e) {
152            throw new RuntimeException(e);
153        }
154        item.setTags(new ArrayList<String>())
155                .setElementById(new HashMap<>())
156                .setElements(new ArrayList<>())
157                .setPublishDate(ZonedDateTime.of(LocalDateTime.now(), ZoneId.of("UTC")))
158                .setTitle("")
159                .setDraft(false)
160                .setTemplate("")
161                .setContent("")
162                .setSlug("")
163        ;
164
165        /* Get the id and slug */
166
167        item.setSlug(FilenameUtils.removeExtension(relativePath));
168        if (!item.getSlug().startsWith("/")) {
169            item.setSlug("/" + item.getSlug());
170        }
171        if (item.getSlug().endsWith("/index")) {
172            item.setSlug(item.getSlug().substring(item.getSlug().length()-6));
173        }
174
175        if (empty(fileContent.trim())) {
176            return item;
177        }
178
179        /* Parse out toml properties, if found */
180        String tomlContent;
181        Matcher tomlMatcher = tomlPattern.matcher(fileContent);
182        if (tomlMatcher.find()) {
183            tomlContent = tomlMatcher.group(1).trim();
184            fileContent = tomlMatcher.replaceAll("\n");
185            Map tomlMap = new Toml().read(tomlContent).to(HashMap.class);
186            for (Object key: tomlMap.keySet()) {
187                Object value = tomlMap.get(key);
188                setProperty(item, key.toString(), value);
189            }
190        }
191
192
193        List<String> allLines = Arrays.asList(fileContent.split("\n"));
194
195
196        if (allLines.size() == 0) {
197            return item;
198        }
199
200        if (empty(item.getTitle())) {
201            item.setTitle(allLines.get(0));
202        }
203
204        String titleLine = "";
205        List<String> propertiesSection = list();
206        String rawContent = "";
207        int propertiesStartAt = 0;
208        if (allLines.size() > 1) {
209            if (allLines.get(1).startsWith("----") || allLines.get(1).startsWith("====")) {
210                titleLine = allLines.get(0);
211                propertiesStartAt = 2;
212                item.setTitle(titleLine);
213            }
214        }
215
216
217        int propertiesEndAt = propertiesStartAt;
218        for (;propertiesEndAt<allLines.size();propertiesEndAt++) {
219            String line = allLines.get(propertiesEndAt);
220            if (line.trim().equals("---")) {
221                continue;
222            }
223            int colon = line.indexOf(':');
224            if (colon == -1) {
225                break;
226            }
227            String key = line.substring(0, colon).trim();
228            String value = line.substring(colon + 1, line.length()).trim();
229            if ("oldUrls".equals(key)) {
230                setProperty(item, key, apply(list(StringUtils.split(value, ";")), (aUrl) ->aUrl.trim()));
231            } else {
232                setProperty(item, key, value);
233            }
234        }
235        if (propertiesEndAt < allLines.size()) {
236            rawContent = StringUtils.join(allLines.subList(propertiesEndAt, allLines.size()), "\n").trim();
237        }
238
239        Boolean isMarkdown = false;
240        if (path.toString().toLowerCase().endsWith(".txt") || path.toString().toLowerCase().endsWith(".md")) {
241            isMarkdown = true;
242        }
243        item.setElements(StElementParser.parseElements(rawContent, isMarkdown));
244        List<StElement> items = item.getElements();
245        for (StElement ele : items) {
246            item.getElementById().put(ele.getId(), ele);
247        }
248
249        String itemContent = StElementParser.removeTags(rawContent).trim();
250        item.setOriginalContent(itemContent);
251
252
253
254        if (isMarkdown) {
255            Log.fine("Parse for page {0} {1} {2}", item.getId(), item.getSlug(), item.getTitle());
256            String cacheKey = DigestUtils.md5Hex("markdown-to-html" + Literals.GSEP + itemContent);
257            String cached = null;
258            if (!"test".equals(Settings.instance().getEnv())) {
259                cached = PermaCache.get(cacheKey);
260            }
261            if (cached == null) {
262                itemContent = Markdown.instance().process(itemContent);
263                PermaCache.set(cacheKey, itemContent);
264            } else {
265                itemContent = cached;
266            }
267
268            item.setContent(itemContent);
269        }
270
271        if (empty(item.getId())) {
272            item.setId(makeIdFromFilePath(relativePath));
273        }
274
275        Log.fine("Loaded text item: id:{0} slug:{1} title:{2} draft:{3}", item.getId(), item.getSlug(), item.getTitle(), item.getDraft());
276        return item;
277    }
278
279
280    public T fromHtml(String fileContent, Path fullPath) {
281
282        String relativePath = fullPath.toString().replace(getBucketFolderPath(), "");
283        Path path = fullPath;
284
285
286        List<String> htmlLines = list();
287        List<String> tomlLines = list();
288        boolean inToml = false;
289        for (String line: StringUtils.split(fileContent.trim(), "\n")) {
290            if (line.trim().equals("<!--start-toml")) {
291                inToml = true;
292                continue;
293            } else if (line.trim().equals("end-toml-->")) {
294                inToml = false;
295                continue;
296            }
297            if (inToml) {
298                tomlLines.add(line);
299            } else {
300                htmlLines.add(line);
301            }
302        }
303
304        String toml = StringUtils.join(tomlLines, "\n");
305        String html = StringUtils.join(htmlLines, "\n");
306
307        T item = null;
308        if (!empty(toml)) {
309            item = new Toml().read(toml).to(getModelClass());
310        } else {
311            try {
312                item = getModelClass().newInstance();
313            } catch (InstantiationException e) {
314                throw new RuntimeException(e);
315            } catch (IllegalAccessException e) {
316                throw new RuntimeException(e);
317            }
318            item.setTags(new ArrayList<String>())
319                    .setElementById(new HashMap<>())
320                    .setElements(new ArrayList<>())
321                    .setTitle("")
322                    .setDraft(true)
323                    .setTemplate("")
324                    .setContent("")
325                    .setSlug(null)
326            ;
327        }
328
329        /* Set the content */
330        item.setContent(html);
331        item.setOriginalContent(fileContent);
332
333        /* Set the slug from the file path, if it does not exist */
334
335        if (item.getSlug() == null) {
336            item.setSlug(FilenameUtils.removeExtension(relativePath));
337            if (!item.getSlug().startsWith("/")) {
338                item.setSlug("/" + item.getSlug());
339            }
340            if (item.getSlug().endsWith("/index")) {
341                item.setSlug(item.getSlug().substring(item.getSlug().length() - 6));
342            }
343        }
344
345        Log.fine("Loaded text item: id:{0} slug:{1} title:{2} draft:{3}", item.getId(), item.getSlug(), item.getTitle(), item.getDraft());
346        return item;
347    }
348
349
350    public String makePathForObject(T obj) {
351        String slug = obj.getSlug();
352        if (empty(slug)) {
353            slug = obj.getId().toString();
354        }
355        return GeneralUtils.slugify(slug).replace('/', '-') + ".txt";
356    }
357
358
359
360
361    protected void setProperty(TextItem item, String key, Object value) {
362        if (key.equals("slug")) {
363            item.setSlug(value.toString());
364        } else if (key.equals("title")) {
365            item.setTitle(value.toString());
366        } else if (key.equals("publishDate")) {
367            for(DateTimeFormatter formatter: localDateFormats) {
368                if (item.getSlug().equals("/future-dated")) {
369                    Log.info("future");
370                }
371                try {
372                    LocalDateTime dt = LocalDateTime.parse(value.toString(), formatter);
373                    ZoneId zoneId = ZoneId.systemDefault();
374                    if (Context.getSettings() != null && Context.getSettings().getTimeZoneId() != null) {
375                        zoneId = Context.getSettings().getTimeZoneId();
376                    }
377                    item.setPublishDate(ZonedDateTime.of(dt, zoneId));
378                    return;
379                } catch (DateTimeParseException e) {
380
381                }
382            }
383            for(DateTimeFormatter formatter: zonedDateFormats) {
384                try {
385                    ZonedDateTime dt = ZonedDateTime.parse(value.toString(), formatter);
386                    item.setPublishDate(dt);
387                    return;
388                } catch (DateTimeParseException e) {
389
390                }
391            }
392
393        } else if (key.equals("draft")) {
394            item.setDraft(value.equals("true"));
395        } else if (key.equals("template")) {
396            item.setTemplate(value.toString());
397        } else if (key.equals("author")) {
398            item.setAuthor(value.toString());
399        } else if (key.equals("tags")) {
400            if (value instanceof List) {
401                item.setTags((List<String>) value);
402            } else {
403                ArrayList<String> tags = new ArrayList<String>();
404                for (String tag : value.toString().split("(;|,)")) {
405                    tags.add(tag.trim());
406                }
407                item.setTags(tags);
408
409            }
410        } else if (key.equals("contentType")) {
411            item.setContentType(value.toString());
412        } else {
413            item.put(key, value);
414        }
415
416    }
417
418
419    public String stringify(TextItem obj)  {
420
421        StringBuffer buffer = new StringBuffer();
422
423        if (obj.getTitle() != null) {
424            buffer.append(String.format("title: %s\n", obj.getTitle()));
425        }
426        if (obj.getSlug() != null) {
427            buffer.append(String.format("slug: %s\n", obj.getSlug()));
428        }
429        if (obj.getPublishDate() != null) {
430            buffer.append(String.format("publishDate: %s\n", obj.getPublishDate().format(zonedDateFormats[0])));
431        }
432        if (obj.getDraft() != null) {
433            buffer.append(String.format("isDraft: %s\n", obj.getDraft().toString().toLowerCase()));
434        }
435        if (obj.getTags() != null && obj.getTags().size() > 0) {
436            buffer.append(String.format("tags: %s\n", StringUtils.join(obj.getTags(), ",")));
437        }
438        if (!StringUtils.isEmpty(obj.getAuthor())) {
439            buffer.append(String.format("author: %s\n", obj.getAuthor()));
440        }
441
442        for(Map.Entry<String, Object> attr: obj.getAttributes().entrySet()) {
443            buffer.append(String.format("%s: %s\n", attr.getKey(), attr.getValue()));
444        }
445
446        if (obj.getElements() != null) {
447            for (StElement element : obj.getElements()) {
448                // TODO: write the attributes, write the tag, write anything else
449                buffer.append(String.format("\n<st-element id=\"%s\">%s</st-element>\n", element.getId(), element.getRawInnerContent()));
450            }
451        }
452
453        if (obj.getOriginalContent() != null) {
454            buffer.append(String.format("\n%s", obj.getOriginalContent()));
455        }
456
457        return buffer.toString();
458
459    }
460
461    public String toHtml(TextItem obj) {
462        TomlWriter writer = new TomlWriter();
463        String html = obj.getContent();
464        // Don't want to mutate the original
465        TextItem clone = JSON.parse(JSON.stringify(obj), obj.getClass());
466        clone.setContent(null);
467        String toml = writer.write(obj);
468
469        html = "<!--start-toml\n" + toml + "\n--end-toml-->\n\n" + html;
470        return html;
471    }
472
473
474
475}