Source code

001/*
002 * Stallion Core: A Modern Web Framework
003 *
004 * Copyright (C) 2015 - 2016 Stallion Software LLC.
005 *
006 * This program is free software: you can redistribute it and/or modify it under the terms of the
007 * GNU General Public License as published by the Free Software Foundation, either version 2 of
008 * the License, or (at your option) any later version. This program is distributed in the hope that
009 * it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
010 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
011 * License for more details. You should have received a copy of the GNU General Public License
012 * along with this program.  If not, see <http://www.gnu.org/licenses/gpl-2.0.html>.
013 *
014 *
015 *
016 */
017
018package io.stallion.dataAccess.file;
019
020import io.stallion.dataAccess.*;
021import io.stallion.dataAccess.db.DefaultSort;
022import io.stallion.dataAccess.filtering.FilterChain;
023import io.stallion.exceptions.ConfigException;
024import io.stallion.exceptions.UsageException;
025import io.stallion.fileSystem.FileSystemWatcherService;
026import io.stallion.fileSystem.TreeVisitor;
027import io.stallion.reflection.PropertyComparator;
028import io.stallion.services.Log;
029import io.stallion.settings.Settings;
030import org.apache.commons.codec.digest.DigestUtils;
031import org.apache.commons.io.FilenameUtils;
032import org.apache.commons.lang3.ArrayUtils;
033import org.apache.commons.lang3.StringUtils;
034
035import java.io.File;
036import java.io.IOException;
037import java.nio.ByteBuffer;
038import java.nio.file.FileSystems;
039import java.nio.file.Files;
040import java.nio.file.Path;
041import java.util.*;
042
043import static io.stallion.utils.Literals.*;
044
045/**
046 * A base persister that handles retrieving and saving model objects to the file sytem.
047 *
048 * @param <T>
049 */
050public abstract class FilePersisterBase<T extends Model> extends BasePersister<T> {
051
052    private String bucketFolderPath = "";
053    private Map<String, Long> fileToIdMap = new HashMap<>();
054    private Map<String, Long> fileToTimestampMap = new HashMap<>();
055    private Map<Long, String> idToFileMap = new HashMap<>();
056    private boolean manyItemsPerFile = false;
057    private String itemArrayName = "";
058    protected String sortField = "lastModifiedMillis";
059    protected String sortDirection = "DESC";
060
061
062    @Override
063    public void init(DataAccessRegistration registration, ModelController<T> controller, Stash<T> stash) {
064        super.init(registration, controller, stash);
065        bucketFolderPath = registration.getAbsolutePath();
066        manyItemsPerFile = registration.isMultiplePerFile();
067        itemArrayName = registration.getItemArrayName();
068        idToFileMap = new HashMap<>();
069        fileToIdMap = new HashMap<>();
070
071        if (!StringUtils.isEmpty(registration.getAbsolutePath())) {
072            Boolean exists = new File(registration.getAbsolutePath()).isDirectory();
073            Log.fine("DAL target {0} exists? {1}", registration.getAbsolutePath(), exists);
074            if (!exists) {
075                new File(registration.getAbsolutePath()).mkdirs();
076            }
077        }
078
079        DefaultSort defaultSort = getModelClass().getAnnotation(DefaultSort.class);
080        if (defaultSort != null) {
081            sortField = defaultSort.field();
082            sortDirection = defaultSort.direction();
083        }
084
085    }
086
087    public abstract Set<String> getFileExtensions();
088
089    public boolean matchesExtension(String path) {
090        String extension = FilenameUtils.getExtension(path).toLowerCase();
091        return getFileExtensions().contains(extension);
092    }
093
094    @Override
095    public List<T> fetchAll()  {
096        File target = new File(Settings.instance().getTargetFolder());
097        if (!target.isDirectory()) {
098            if (getItemController().isWritable()) {
099                target.mkdirs();
100            } else {
101                throw new ConfigException(String.format("The JSON bucket %s (path %s) is read-only, but does not exist in the file system. Either create the folder, make it writable, or remove it from the configuration.", getItemController().getBucket(), getBucketFolderPath()));
102            }
103        }
104        TreeVisitor visitor = new TreeVisitor();
105        Path folderPath = FileSystems.getDefault().getPath(getBucketFolderPath());
106        try {
107            Files.walkFileTree(folderPath, visitor);
108        } catch (IOException e) {
109            throw new RuntimeException(e);
110        }
111        List<T> objects = new ArrayList<>();
112        for (Path path : visitor.getPaths()) {
113            if (!matchesExtension(path.toString())) {
114                continue;
115            }
116
117            if (path.toString().contains(".#")) {
118                continue;
119            }
120            if (path.getFileName().startsWith(".")) {
121                continue;
122            }
123            T o = fetchOne(path.toString());
124            if (o != null) {
125                objects.add(o);
126            }
127
128        }
129        objects.sort(new PropertyComparator<T>(sortField));
130        if (sortDirection.toLowerCase().equals("desc")) {
131            Collections.reverse(objects);
132        }
133
134        return objects;
135    }
136
137    @Override
138    public T fetchOne(T obj) {
139        return fetchOne(fullFilePathForObj(obj));
140    }
141
142
143    @Override
144    public T fetchOne(Long id) {
145        return fetchOne(fullFilePathForId(id));
146    }
147
148    public T fetchOne(String filePath) {
149        if (filePath.startsWith(".") || filePath.startsWith("#") || filePath.contains("..")) {
150            return null;
151        }
152        if (!matchesExtension(filePath)) {
153            return null;
154        }
155        File file = new File(filePath);
156        if (!file.exists() || !file.isFile()) {
157            return null;
158        }
159        T o = doFetchOne(file);
160        if (o == null) {
161            return null;
162        }
163        if (empty(o.getId())) {
164            o.setId(makeIdFromFilePath(filePath));
165        }
166
167        Long ts = file.lastModified();
168        o.setLastModifiedMillis(ts);
169
170        handleFetchOne(o);
171        onPostLoadFromFile(o, filePath);
172        return o;
173    }
174
175    public abstract T doFetchOne(File file);
176
177
178    public FilterChain<T> filterChain() {
179        throw new UsageException("File based persistence does not work with filter chains. You have to use a LocalStash, which will provide an in memory filter chain.");
180    }
181
182    @Override
183    public void hardDelete(T obj)  {
184        String filePath = fullFilePathForObj(obj);
185        File file = new File(filePath);
186        file.delete();
187    }
188
189
190    public void onPostLoadFromFile(T obj, String path) {
191        if (path.startsWith(getBucketFolderPath())) {
192            path = path.replace(getBucketFolderPath(), "");
193        }
194        if (path.startsWith("/")) {
195            path = StringUtils.stripStart(path, "/");
196        }
197        getIdToFileMap().put(obj.getId(), path);
198        getFileToIdMap().put(path, obj.getId());
199        if (obj instanceof ModelWithFilePath) {
200            ((ModelWithFilePath) obj).setFilePath(path);
201        }
202
203    }
204
205    public boolean reloadIfNewer(T obj) {
206        String path = fullFilePathForObj(obj);
207        File file = new File(path.toString());
208        Long currentTs = file.lastModified();
209        Long fileLastModified = or(obj.getLastModifiedMillis(), 0L);
210        if (currentTs >= fileLastModified) {
211            getStash().loadForId(obj.getId());
212            fileToTimestampMap.put(path.toString(), currentTs);
213            return true;
214        }
215        return false;
216    }
217
218    public String relativeFilePathForObj(T obj) {
219        if (obj instanceof ModelWithFilePath) {
220            if (!empty(((ModelWithFilePath) obj).getFilePath())) {
221                return ((ModelWithFilePath) obj).getFilePath();
222            } else {
223                String path = ((ModelWithFilePath) obj).generateFilePath();
224                ((ModelWithFilePath) obj).setFilePath(path);
225                return path;
226            }
227        } else if (getIdToFileMap().containsKey(obj.getId())) {
228            return getIdToFileMap().get(obj.getId());
229        } else {
230            return makePathForObject(obj);
231        }
232    }
233
234    public String makePathForObject(T obj) {
235        return obj.getId().toString() + ".json";
236    }
237
238    public String fullFilePathForObj(T obj) {
239        String path = getBucketFolderPath();
240        if (!path.endsWith("/")) {
241            path += "/";
242        }
243        return path + relativeFilePathForObj(obj);
244    }
245
246    public String fullFilePathForId(Long id) {
247        String path = getBucketFolderPath();
248        if (!path.endsWith("/")) {
249            path += "/";
250        }
251        if (!getIdToFileMap().containsKey(id)) {
252            return null;
253        }
254        return path + getIdToFileMap().get(id);
255    }
256
257    public void watchEventCallback(String filePath) {
258        if (getFileToIdMap().containsKey(filePath)) {
259            getStash().loadForId(getFileToIdMap().get(filePath));
260        } else {
261            String fullPath = getBucketFolderPath() + "/" + filePath;
262            T o = fetchOne(fullPath);
263            if (o != null) {
264                getStash().loadItem(o);
265            }
266        }
267    }
268
269    @Override
270    public void attachWatcher()  {
271        FileSystemWatcherService.instance().registerWatcher(
272                new ItemFileChangeEventHandler(this)
273                        .setWatchedFolder(this.getBucketFolderPath())
274                        .setWatchTree(true)
275        );
276    }
277
278    /**
279     * Derives a Long id by hashing the file path and then taking the first 8 bytes
280     * of the path.
281     *
282     * This is used if the model object doesn't have a defined id field.
283     *
284     * @param path
285     * @return
286     */
287    public Long makeIdFromFilePath(String path) {
288        path = path.toLowerCase();
289        path = path.replace(getBucketFolderPath().toLowerCase(), "");
290        path = StringUtils.stripStart(path, "/");
291        path = getBucket() + "-----" + path;
292        // Derive a long id by hashing the file path
293        byte[] bs = Arrays.copyOfRange(DigestUtils.md5(path), 0, 6);
294        bs = ArrayUtils.addAll(new byte[]{0,0}, bs);
295        ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
296        buffer.put(bs);
297        buffer.flip();//need flip
298        Long l = buffer.getLong();
299        if (l < 0) {
300            l = -l;
301        }
302        Log.finest("calculated id is {0}", l);
303        return l;
304    }
305
306
307    public String getBucketFolderPath() {
308        return bucketFolderPath;
309    }
310
311    public FilePersisterBase setBucketFolderPath(String bucketFolderPath) {
312        this.bucketFolderPath = bucketFolderPath;
313        return this;
314    }
315
316
317    public Map<String, Long> getFileToIdMap() {
318        return fileToIdMap;
319    }
320
321    public FilePersisterBase setFileToIdMap(Map<String, Long> fileToIdMap) {
322        this.fileToIdMap = fileToIdMap;
323        return this;
324    }
325
326
327    public Map<Long, String> getIdToFileMap() {
328        return idToFileMap;
329    }
330
331    public FilePersisterBase setIdToFileMap(Map<Long, String> idToFileMap) {
332        this.idToFileMap = idToFileMap;
333        return this;
334    }
335
336    public boolean isManyItemsPerFile() {
337        return manyItemsPerFile;
338    }
339
340    public FilePersisterBase setManyItemsPerFile(boolean manyItemsPerFile) {
341        this.manyItemsPerFile = manyItemsPerFile;
342        return this;
343    }
344
345    public String getItemArrayName() {
346        return itemArrayName;
347    }
348
349    public FilePersisterBase setItemArrayName(String itemArrayName) {
350        this.itemArrayName = itemArrayName;
351        return this;
352    }
353}